diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index b58c52ea50f5..7e5acd7c52ed 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -5474,4 +5474,759 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx + + <structname>pg_stat_vacuum_database</structname> + + + pg_stat_vacuum_database + + + + The view pg_stat_vacuum_database will contain + one row for each database in the current cluster, showing statistics about + vacuuming that database. + + + + <structname>pg_stat_vacuum_database</structname> Columns + + + + + Column Type + + + Description + + + + + + + + dbid oid + + + OID of a database + + + + + + total_blks_read int8 + + + Number of database blocks read by vacuum operations + performed on this database + + + + + + total_blks_hit int8 + + + Number of times database blocks were found in the + buffer cache by vacuum operations + performed on this database + + + + + + total_blks_dirtied int8 + + + Number of database blocks dirtied by vacuum operations + performed on this database + + + + + + total_blks_written int8 + + + Number of database blocks written by vacuum operations + performed on this database + + + + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations + performed on this database + + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + performed on this database + + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations + performed on this database + + + + + + blk_read_time float8 + + + Time spent reading database blocks by vacuum operations performed on + this database, in milliseconds (if is enabled, + otherwise zero) + + + + + + blk_write_time float8 + + + Time spent writing database blocks by vacuum operations performed on + this database, in milliseconds (if is enabled, + otherwise zero) + + + + + + delay_time float8 + + + Time spent sleeping in a vacuum delay point by vacuum operations performed on + this database, in milliseconds (see + for details) + + + + + + system_time float8 + + + System CPU time of vacuuming this database, in milliseconds + + + + + + user_time float8 + + + User CPU time of vacuuming this database, in milliseconds + + + + + + total_time float8 + + + Total time of vacuuming this database, in milliseconds + + + + + + wraparound_failsafe_count int4 + + + Number of times the vacuum was run to prevent a wraparound problem. + + + + + + errors int4 + + + Number of times vacuum operations performed on this database + were interrupted on any errors + + + + +
+
+ + + <structname>pg_stat_vacuum_indexes</structname> + + + pg_stat_vacuum_indexes + + + + The view pg_stat_vacuum_indexes will contain + one row for each index in the current database (including TOAST + table indexes), showing statistics about vacuuming that specific index. + + + + <structname>pg_stat_vacuum_indexes</structname> Columns + + + + + Column Type + + + Description + + + + + + + + relid oid + + + OID of an index + + + + + + schema name + + + Name of the schema this index is in + + + + + + relname name + + + Name of this index + + + + + + total_blks_read int8 + + + Number of database blocks read by vacuum operations + performed on this index + + + + + + total_blks_hit int8 + + + Number of times database blocks were found in the + buffer cache by vacuum operations + performed on this index + + + + + + total_blks_dirtied int8 + + + Number of database blocks dirtied by vacuum operations + performed on this index + + + + + + total_blks_written int8 + + + Number of database blocks written by vacuum operations + performed on this index + + + + + + rel_blks_read int8 + + + Number of blocks vacuum operations read from this + index + + + + + + rel_blks_hit int8 + + + Number of times blocks of this index were already found + in the buffer cache by vacuum operations, so that a read was not necessary + (this only includes hits in the + project; buffer cache, not the operating system's file system cache) + + + + + + pages_deleted int8 + + + Number of pages deleted by vacuum operations + performed on this index + + + + + + tuples_deleted int8 + + + Number of dead tuples vacuum operations deleted from this index + + + + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations + performed on this index + + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + performed on this index + + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations + performed on this index + + + + + + blk_read_time int8 + + + Time spent reading database blocks by vacuum operations performed on + this index, in milliseconds (if is enabled, + otherwise zero) + + + + + + blk_write_time int8 + + + Time spent writing database blocks by vacuum operations performed on + this index, in milliseconds (if is enabled, + otherwise zero) + + + + + + delay_time float8 + + + Time spent sleeping in a vacuum delay point by vacuum operations performed on + this index, in milliseconds (see + for details) + + + + + + system_time float8 + + + System CPU time of vacuuming this index, in milliseconds + + + + + + user_time float8 + + + User CPU time of vacuuming this index, in milliseconds + + + + + + total_time float8 + + + Total time of vacuuming this index, in milliseconds + + + + + +
+
+ + + <structname>pg_stat_vacuum_tables</structname> + + + pg_stat_vacuum_tables + + + + The view pg_stat_vacuum_tables will contain + one row for each table in the current database (including TOAST + tables), showing statistics about vacuuming that specific table. + + + + <structname>pg_stat_vacuum_tables</structname> Columns + + + + + Column Type + + + Description + + + + + + + + relid oid + + + OID of a table + + + + + + schema name + + + Name of the schema this table is in + + + + + + relname name + + + Name of this table + + + + + + total_blks_read int8 + + + Number of database blocks read by vacuum operations + performed on this table + + + + + + total_blks_hit int8 + + + Number of times database blocks were found in the + buffer cache by vacuum operations + performed on this table + + + + + + total_blks_dirtied int8 + + + Number of blocks written directly by vacuum or auto vacuum. + Blocks that are dirtied by a vacuum process can be written out by another process. + + + + + + total_blks_written int8 + + + Number of database blocks written by vacuum operations + performed on this table + + + + + + rel_blks_read int8 + + + Number of blocks vacuum operations read from this + table + + + + + + rel_blks_hit int8 + + + Number of times blocks of this table were already found + in the buffer cache by vacuum operations, so that a read was not necessary + (this only includes hits in the + project; buffer cache, not the operating system's file system cache) + + + + + + pages_scanned int8 + + + Number of pages examined by vacuum operations + performed on this table + + + + + + pages_removed int8 + + + Number of pages removed from the physical storage by vacuum operations + performed on this table + + + + + + vm_new_frozen_pages int8 + + + Number of the number of pages newly set all-frozen by vacuum + in the visibility map. + + + + + + vm_new_visible_pages int8 + + + Number of the number of pages newly set all-visible by vacuum + in the visibility map. + + + + + + vm_new_visible_frozen_pages int8 + + + Number of the number of pages newly set all-visible and all-frozen + by vacuum in the visibility map. + + + + + + tuples_deleted int8 + + + Number of dead tuples vacuum operations deleted from this table + + + + + + tuples_frozen int8 + + + Number of tuples of this table that vacuum operations marked as + frozen + + + + + + recently_dead_tuples int8 + + + Number of dead tuples vacuum operations left in this table due + to their visibility in transactions + + + + + + missed_dead_tuples int8 + + + Number of fully DEAD (not just RECENTLY_DEAD) tuples that could not be + pruned due to failure to acquire a cleanup lock on a heap page. + + + + + + index_vacuum_count int8 + + + Number of times indexes on this table were vacuumed + + + + + + wraparound_failsafe_count int4 + + + Number of times the vacuum was run to prevent a wraparound problem. + + + + + + missed_dead_pages int8 + + + Number of pages that had at least one missed_dead_tuples. + + + + + + wal_records int8 + + + Total number of WAL records generated by vacuum operations + performed on this table + + + + + + wal_fpi int8 + + + Total number of WAL full page images generated by vacuum operations + performed on this table + + + + + + wal_bytes numeric + + + Total amount of WAL bytes generated by vacuum operations + performed on this table + + + + + + blk_read_time int8 + + + Time spent reading database blocks by vacuum operations performed on + this table, in milliseconds (if is enabled, + otherwise zero) + + + + + + blk_write_time int8 + + + Time spent writing database blocks by vacuum operations performed on + this table, in milliseconds (if is enabled, + otherwise zero) + + + + + + delay_time float8 + + + Time spent sleeping in a vacuum delay point by vacuum operations performed on + this table, in milliseconds (see + for details) + + + + + + system_time float8 + + + System CPU time of vacuuming this table, in milliseconds + + + + + + user_time float8 + + + User CPU time of vacuuming this table, in milliseconds + + + + + + total_time float8 + + + Total time of vacuuming this table, in milliseconds + + + + + +
+ Columns total_*, wal_* + and blk_* include data on vacuuming indexes on this table, while columns + system_time and user_time only include data + on vacuuming the heap. +
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index f28326bad095..8a328638fd34 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -290,6 +290,8 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -408,6 +410,10 @@ typedef struct LVRelState * been permanently disabled. */ BlockNumber eager_scan_remaining_fails; + + int32 wraparound_failsafe_count; /* number of emergency vacuums to prevent anti-wraparound shutdown */ + + PgStat_VacuumRelationCounts extVacReport; } LVRelState; @@ -419,7 +425,6 @@ typedef struct LVSavedErrInfo VacErrPhase phase; } LVSavedErrInfo; - /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); static void heap_vacuum_eager_scan_setup(LVRelState *vacrel, @@ -475,6 +480,188 @@ static void update_vacuum_error_info(LVRelState *vacrel, static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* ---------- + * extvac_stats_start() - + * + * Save cut-off values of extended vacuum counters before start of a relation + * processing. + * ---------- + */ +static void +extvac_stats_start(Relation rel, LVExtStatCounters *counters) +{ + TimestampTz starttime; + + if(!pgstat_track_vacuum_statistics) + return; + + memset(counters, 0, sizeof(LVExtStatCounters)); + + starttime = GetCurrentTimestamp(); + + counters->starttime = starttime; + counters->walusage = pgWalUsage; + counters->bufusage = pgBufferUsage; + counters->VacuumDelayTime = VacuumDelayTime; + counters->blocks_fetched = 0; + counters->blocks_hit = 0; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or user doesn't want to track a database + * activity - just suppress it. + */ + return; + + counters->blocks_fetched = rel->pgstat_info->counts.blocks_fetched; + counters->blocks_hit = rel->pgstat_info->counts.blocks_hit; +} + +/* ---------- + * extvac_stats_end() - + * + * Called to finish an extended vacuum statistic gathering and form a report. + * ---------- + */ +static void +extvac_stats_end(Relation rel, LVExtStatCounters *counters, + PgStat_VacuumRelationCounts *report) +{ + WalUsage walusage; + BufferUsage bufusage; + TimestampTz endtime; + long secs; + int usecs; + + if(!pgstat_track_vacuum_statistics) + return; + + /* Calculate diffs of global stat parameters on WAL and buffer usage. */ + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); + + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + endtime = GetCurrentTimestamp(); + TimestampDifference(counters->starttime, endtime, &secs, &usecs); + + /* + * Fill additional statistics on a vacuum processing operation. + */ + report->total_blks_read += bufusage.local_blks_read + bufusage.shared_blks_read; + report->total_blks_hit += bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->total_blks_dirtied += bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->total_blks_written += bufusage.shared_blks_written; + + report->wal_records += walusage.wal_records; + report->wal_fpi += walusage.wal_fpi; + report->wal_bytes += walusage.wal_bytes; + + report->blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage.local_blk_read_time); + report->blk_read_time += INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_read_time); + report->blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage.local_blk_write_time); + report->blk_write_time += INSTR_TIME_GET_MILLISEC(bufusage.shared_blk_write_time); + report->delay_time += VacuumDelayTime - counters->VacuumDelayTime; + + report->total_time += secs * 1000. + usecs / 1000.; + + if (!rel->pgstat_info || !pgstat_track_counts) + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + return; + + report->blks_fetched += + rel->pgstat_info->counts.blocks_fetched - counters->blocks_fetched; + report->blks_hit += + rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; +} + +void +extvac_stats_start_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters) +{ + if(!pgstat_track_vacuum_statistics) + return; + + /* Set initial values for common heap and index statistics*/ + extvac_stats_start(rel, &counters->common); + counters->pages_deleted = counters->tuples_removed = 0; + + if (stats != NULL) + { + /* + * XXX: Why do we need this code here? If it is needed, I feel lack of + * comments, describing the reason. + */ + counters->tuples_removed = stats->tuples_removed; + counters->pages_deleted = stats->pages_deleted; + } +} + +void +extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, PgStat_VacuumRelationCounts *report) +{ + memset(report, 0, sizeof(PgStat_VacuumRelationCounts)); + + extvac_stats_end(rel, &counters->common, report); + report->type = PGSTAT_EXTVAC_INDEX; + + if (stats != NULL) + { + /* + * if something goes wrong or an user doesn't want to track a database + * activity - just suppress it. + */ + + /* Fill index-specific extended stats fields */ + report->tuples_deleted = + stats->tuples_removed - counters->tuples_removed; + report->index.pages_deleted = + stats->pages_deleted - counters->pages_deleted; + } +} + +/* Accumulate vacuum statistics for heap. + * + * Because of complexity of vacuum processing: it switch procesing between + * the heap relation to index relations and visa versa, we need to store + * gathered statistics information for heap relations several times before + * the vacuum starts processing the indexes again. + * + * It is necessary to gather correct statistics information for heap and indexes + * otherwice the index statistics information would be added to his parent heap + * statistics information and it would be difficult to analyze it later. + * + * We can't subtract union vacuum statistics information for index from the heap relations + * because of total and delay time time statistics collecting during parallel vacuum + * procudure. +*/ +static void +accumulate_heap_vacuum_statistics(LVExtStatCounters *extVacCounters, LVRelState *vacrel) +{ + if (!pgstat_track_vacuum_statistics) + return; + + /* Fill heap-specific extended stats fields */ + vacrel->extVacReport.type = PGSTAT_EXTVAC_TABLE; + vacrel->extVacReport.table.pages_scanned += vacrel->scanned_pages; + vacrel->extVacReport.table.pages_removed += vacrel->removed_pages; + vacrel->extVacReport.table.vm_new_frozen_pages += vacrel->vm_new_frozen_pages; + vacrel->extVacReport.table.vm_new_visible_pages += vacrel->vm_new_visible_pages; + vacrel->extVacReport.table.vm_new_visible_frozen_pages += vacrel->vm_new_visible_frozen_pages; + vacrel->extVacReport.tuples_deleted += vacrel->tuples_deleted; + vacrel->extVacReport.table.tuples_frozen += vacrel->tuples_frozen; + vacrel->extVacReport.table.recently_dead_tuples += vacrel->recently_dead_tuples; + vacrel->extVacReport.table.recently_dead_tuples += vacrel->recently_dead_tuples; + vacrel->extVacReport.table.missed_dead_tuples += vacrel->missed_dead_tuples; + vacrel->extVacReport.table.missed_dead_pages += vacrel->missed_dead_pages; + vacrel->extVacReport.table.index_vacuum_count += vacrel->num_index_scans; + vacrel->extVacReport.wraparound_failsafe_count += vacrel->wraparound_failsafe_count; +} /* @@ -632,6 +819,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; + LVExtStatCounters extVacCounters; char **indnames = NULL; verbose = (params->options & VACOPT_VERBOSE) != 0; @@ -652,7 +840,6 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); - /* * Setup error traceback support for ereport() first. The idea is to set * up an error context callback to display additional information on any @@ -669,6 +856,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -758,6 +946,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->vm_new_visible_frozen_pages = 0; vacrel->vm_new_frozen_pages = 0; vacrel->rel_pages = orig_rel_pages = RelationGetNumberOfBlocks(rel); + vacrel->wraparound_failsafe_count = 0; /* * Get cutoffs that determine which deleted tuples are considered DEAD, @@ -837,6 +1026,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, */ lazy_scan_heap(vacrel); + extvac_stats_start(rel, &extVacCounters); + /* * Free resources managed by dead_items_alloc. This ends parallel mode in * passing when necessary. @@ -933,13 +1124,22 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * It seems like a good idea to err on the side of not vacuuming again too * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. + * + * We are ready to send vacuum statistics information for heap relations. */ + pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + vacrel->missed_dead_tuples, starttime); + + /* Make generic extended vacuum stats report and fill heap-specific extended stats fields */ + extvac_stats_end(vacrel->rel, &extVacCounters, &(vacrel->extVacReport)); + accumulate_heap_vacuum_statistics(&extVacCounters, vacrel); + pgstat_report_tab_vacuum_extstats(vacrel->reloid, rel->rd_rel->relisshared, &(vacrel->extVacReport)); + pgstat_progress_end_command(); if (instrument) @@ -1211,6 +1411,7 @@ lazy_scan_heap(LVRelState *vacrel) PROGRESS_VACUUM_MAX_DEAD_TUPLE_BYTES }; int64 initprog_val[3]; + LVExtStatCounters extVacCounters; /* Report that we're scanning the heap, advertising total # of blocks */ initprog_val[0] = PROGRESS_VACUUM_PHASE_SCAN_HEAP; @@ -1225,6 +1426,13 @@ lazy_scan_heap(LVRelState *vacrel) vacrel->next_unskippable_eager_scanned = false; vacrel->next_unskippable_vmbuffer = InvalidBuffer; + /* + * Due to the fact that vacuum heap processing needs their index vacuuming + * we need to track them separately and accumulate heap vacuum statistics + * separately. So last processes are related to only heap vacuuming process. + */ + extvac_stats_start(vacrel->rel, &extVacCounters); + /* * Set up the read stream for vacuum's first pass through the heap. * @@ -1289,8 +1497,26 @@ lazy_scan_heap(LVRelState *vacrel) /* Perform a round of index and heap vacuuming */ vacrel->consider_bypass_optimization = false; + + /* + * Lazy vacuum stage includes index vacuuming and cleaning up stage, so + * we prefer tracking them separately. + * Before starting to process the indexes save the current heap statistics + */ + extvac_stats_end(vacrel->rel, &extVacCounters, &(vacrel->extVacReport)); + accumulate_heap_vacuum_statistics(&extVacCounters, vacrel); + lazy_vacuum(vacrel); + /* + * After completion lazy vacuum, we start again tracking vacuum statistics for + * heap-related objects like FSM, VM, provide heap prunning. + * It seems dangerously that we have start tracking but there are no end, but + * it is safe. The end tracking is located before lazy vacuum stage in the same + * loop or after it. + */ + extvac_stats_start(vacrel->rel, &extVacCounters); + /* * Vacuum the Free Space Map to make newly-freed space visible on * upper-level FSM pages. Note that blkno is the previously @@ -1513,6 +1739,12 @@ lazy_scan_heap(LVRelState *vacrel) read_stream_end(stream); + /* + * Vacuum can process lazy vacuum again and we save heap statistics now + * just in case in tend to avoid collecting vacuum index statistics again. + */ + extvac_stats_end(vacrel->rel, &extVacCounters, &(vacrel->extVacReport)); + accumulate_heap_vacuum_statistics(&extVacCounters, vacrel); /* * Do index vacuuming (call each index's ambulkdelete routine), then do * related heap vacuuming @@ -1520,6 +1752,12 @@ lazy_scan_heap(LVRelState *vacrel) if (vacrel->dead_items_info->num_items > 0) lazy_vacuum(vacrel); + /* + * We need to take into account heap vacuum statistics during process of + * FSM. + */ + extvac_stats_start(vacrel->rel, &extVacCounters); + /* * Vacuum the remainder of the Free Space Map. We must do this whether or * not there were indexes, and whether or not we bypassed index vacuuming. @@ -1532,6 +1770,10 @@ lazy_scan_heap(LVRelState *vacrel) /* report all blocks vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, rel_pages); + /* Before starting final index clan up stage save heap statistics */ + extvac_stats_end(vacrel->rel, &extVacCounters, &(vacrel->extVacReport)); + accumulate_heap_vacuum_statistics(&extVacCounters, vacrel); + /* Do final index cleanup (call each index's amvacuumcleanup routine) */ if (vacrel->nindexes > 0 && vacrel->do_index_cleanup) lazy_cleanup_all_indexes(vacrel); @@ -2449,6 +2691,7 @@ static void lazy_vacuum(LVRelState *vacrel) { bool bypass; + LVExtStatCounters extVacCounters; /* Should not end up here with no indexes */ Assert(vacrel->nindexes > 0); @@ -2461,6 +2704,9 @@ lazy_vacuum(LVRelState *vacrel) return; } + /* Set initial statistics values to gather vacuum statistics for the heap */ + extvac_stats_start(vacrel->rel, &extVacCounters); + /* * Consider bypassing index vacuuming (and heap vacuuming) entirely. * @@ -2517,6 +2763,14 @@ lazy_vacuum(LVRelState *vacrel) TidStoreMemoryUsage(vacrel->dead_items) < 32 * 1024 * 1024); } + /* + * Vacuum is likely to vacuum indexes again, so save vacuum statistics for + * heap relations now. + * The vacuum process below doesn't contain any useful statistics information + * for heap if indexes won't be processed, but we will track them separately. + */ + extvac_stats_end(vacrel->rel, &extVacCounters, &(vacrel->extVacReport)); + if (bypass) { /* @@ -2533,11 +2787,21 @@ lazy_vacuum(LVRelState *vacrel) } else if (lazy_vacuum_all_indexes(vacrel)) { - /* - * We successfully completed a round of index vacuuming. Do related - * heap vacuuming now. - */ - lazy_vacuum_heap_rel(vacrel); + /* Now the vacuum is going to process heap relation, so + * we need to set intial statistic values for tracking. + */ + + /* Set initial statistics values to gather vacuum statistics for the heap */ + extvac_stats_start(vacrel->rel, &extVacCounters); + + /* + * We successfully completed a round of index vacuuming. Do related + * heap vacuuming now. + */ + lazy_vacuum_heap_rel(vacrel); + + extvac_stats_end(vacrel->rel, &extVacCounters, &(vacrel->extVacReport)); + accumulate_heap_vacuum_statistics(&extVacCounters, vacrel); } else { @@ -2972,6 +3236,7 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel) int64 progress_val[2] = {0, 0}; VacuumFailsafeActive = true; + vacrel->wraparound_failsafe_count ++; /* * Abandon use of a buffer access strategy to allow use of all of @@ -3083,6 +3348,11 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + PgStat_VacuumRelationCounts extVacReport; + + /* Set initial statistics values to gather vacuum statistics for the index */ + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -3101,6 +3371,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -3109,6 +3380,11 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_bulkdel_one_index(&ivinfo, istat, vacrel->dead_items, vacrel->dead_items_info); + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + pgstat_report_tab_vacuum_extstats(vacrel->indoid, indrel->rd_rel->relisshared, + &extVacReport); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3133,6 +3409,11 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCountersIdx extVacCounters; + PgStat_VacuumRelationCounts extVacReport; + + /* Set initial statistics values to gather vacuum statistics for the index */ + extvac_stats_start_idx(indrel, istat, &extVacCounters); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -3152,12 +3433,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat, &extVacCounters, &extVacReport); + pgstat_report_tab_vacuum_extstats(vacrel->indoid, indrel->rd_rel->relisshared, + &extVacReport); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3770,6 +4057,9 @@ vacuum_error_callback(void *arg) switch (errinfo->phase) { case VACUUM_ERRCB_PHASE_SCAN_HEAP: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(); + if (BlockNumberIsValid(errinfo->blkno)) { if (OffsetNumberIsValid(errinfo->offnum)) @@ -3785,6 +4075,9 @@ vacuum_error_callback(void *arg) break; case VACUUM_ERRCB_PHASE_VACUUM_HEAP: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(); + if (BlockNumberIsValid(errinfo->blkno)) { if (OffsetNumberIsValid(errinfo->offnum)) @@ -3800,16 +4093,25 @@ vacuum_error_callback(void *arg) break; case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(); + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", errinfo->indname, errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(); + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", errinfo->indname, errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_TRUNCATE: + if(geterrelevel() == ERROR) + pgstat_report_vacuum_error(); + if (BlockNumberIsValid(errinfo->blkno)) errcontext("while truncating relation \"%s.%s\" to %u blocks", errinfo->relnamespace, errinfo->relname, errinfo->blkno); diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 745a04ef26e2..07623a045fa7 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -91,6 +91,7 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/pg_bitutils.h" #include "storage/bufmgr.h" #include "storage/smgr.h" @@ -160,6 +161,15 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags if (map[mapByte] & mask) { + /* + * As part of vacuum stats, track how often all-visible or all-frozen + * bits are cleared. + */ + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_VISIBLE) + pgstat_count_vm_rev_all_visible(rel); + if (map[mapByte] >> mapOffset & flags & VISIBILITYMAP_ALL_FROZEN) + pgstat_count_vm_rev_all_frozen(rel); + map[mapByte] &= ~mask; MarkBufferDirty(vmbuf); diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index fbaed5359ad7..72c8e339c45e 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -1873,6 +1873,7 @@ heap_drop_with_catalog(Oid relid) /* ensure that stats are dropped if transaction commits */ pgstat_drop_relation(rel); + pgstat_vacuum_relation_delete_pending_cb(RelationGetRelid(rel)); /* * Close relcache entry, but *keep* AccessExclusiveLock on the relation diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 739a92bdcc1c..e4fa754aab42 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -2327,6 +2327,7 @@ index_drop(Oid indexId, bool concurrent, bool concurrent_lock_mode) /* ensure that stats are dropped if transaction commits */ pgstat_drop_relation(userIndexRelation); + pgstat_vacuum_relation_delete_pending_cb(RelationGetRelid(userIndexRelation)); /* * Close and flush the index's relcache entry, to ensure relcache doesn't diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 15efb02badb8..0f8346d7b3c9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -713,7 +713,9 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_total_vacuum_time(C.oid) AS total_vacuum_time, pg_stat_get_total_autovacuum_time(C.oid) AS total_autovacuum_time, pg_stat_get_total_analyze_time(C.oid) AS total_analyze_time, - pg_stat_get_total_autoanalyze_time(C.oid) AS total_autoanalyze_time + pg_stat_get_total_autoanalyze_time(C.oid) AS total_autoanalyze_time, + pg_stat_get_rev_all_frozen_pages(C.oid) AS rev_all_frozen_pages, + pg_stat_get_rev_all_visible_pages(C.oid) AS rev_all_visible_pages FROM pg_class C LEFT JOIN pg_index I ON C.oid = I.indrelid LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) @@ -1412,3 +1414,108 @@ REVOKE ALL ON pg_aios FROM PUBLIC; GRANT SELECT ON pg_aios TO pg_read_all_stats; REVOKE EXECUTE ON FUNCTION pg_get_aios() FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_get_aios() TO pg_read_all_stats; +-- +-- Show extended cumulative statistics on a vacuum operation over all tables and +-- databases of the instance. +-- Use Invalid Oid "0" as an input relation id to get stat on each table in a +-- database. +-- + +CREATE VIEW pg_stat_vacuum_tables AS +SELECT + ns.nspname AS schemaname, + rel.relname AS relname, + stats.relid as relid, + + stats.total_blks_read AS total_blks_read, + stats.total_blks_hit AS total_blks_hit, + stats.total_blks_dirtied AS total_blks_dirtied, + stats.total_blks_written AS total_blks_written, + + stats.rel_blks_read AS rel_blks_read, + stats.rel_blks_hit AS rel_blks_hit, + + stats.pages_scanned AS pages_scanned, + stats.pages_removed AS pages_removed, + stats.vm_new_frozen_pages AS vm_new_frozen_pages, + stats.vm_new_visible_pages AS vm_new_visible_pages, + stats.vm_new_visible_frozen_pages AS vm_new_visible_frozen_pages, + stats.missed_dead_pages AS missed_dead_pages, + stats.tuples_deleted AS tuples_deleted, + stats.tuples_frozen AS tuples_frozen, + stats.recently_dead_tuples AS recently_dead_tuples, + stats.missed_dead_tuples AS missed_dead_tuples, + + stats.wraparound_failsafe AS wraparound_failsafe, + stats.index_vacuum_count AS index_vacuum_count, + stats.wal_records AS wal_records, + stats.wal_fpi AS wal_fpi, + stats.wal_bytes AS wal_bytes, + + stats.blk_read_time AS blk_read_time, + stats.blk_write_time AS blk_write_time, + + stats.delay_time AS delay_time, + stats.total_time AS total_time + +FROM pg_class rel + JOIN pg_namespace ns ON ns.oid = rel.relnamespace, + LATERAL pg_stat_get_vacuum_tables(rel.oid) stats +WHERE rel.relkind = 'r'; + +CREATE VIEW pg_stat_vacuum_indexes AS +SELECT + rel.oid as relid, + ns.nspname AS schemaname, + rel.relname AS relname, + + total_blks_read AS total_blks_read, + total_blks_hit AS total_blks_hit, + total_blks_dirtied AS total_blks_dirtied, + total_blks_written AS total_blks_written, + + rel_blks_read AS rel_blks_read, + rel_blks_hit AS rel_blks_hit, + + pages_deleted AS pages_deleted, + tuples_deleted AS tuples_deleted, + + wal_records AS wal_records, + wal_fpi AS wal_fpi, + wal_bytes AS wal_bytes, + + blk_read_time AS blk_read_time, + blk_write_time AS blk_write_time, + + delay_time AS delay_time, + total_time AS total_time +FROM + pg_class rel + JOIN pg_namespace ns ON ns.oid = rel.relnamespace, + LATERAL pg_stat_get_vacuum_indexes(rel.oid) stats +WHERE rel.relkind = 'i'; + +CREATE VIEW pg_stat_vacuum_database AS +SELECT + db.oid as dboid, + db.datname AS dbname, + + stats.db_blks_read AS db_blks_read, + stats.db_blks_hit AS db_blks_hit, + stats.total_blks_dirtied AS total_blks_dirtied, + stats.total_blks_written AS total_blks_written, + + stats.wal_records AS wal_records, + stats.wal_fpi AS wal_fpi, + stats.wal_bytes AS wal_bytes, + + stats.blk_read_time AS blk_read_time, + stats.blk_write_time AS blk_write_time, + + stats.delay_time AS delay_time, + stats.total_time AS total_time, + stats.wraparound_failsafe AS wraparound_failsafe, + stats.errors AS errors +FROM + pg_database db, + LATERAL pg_stat_get_vacuum_database(db.oid) stats; \ No newline at end of file diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 5fbbcdaabb1d..c4b910cd9282 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1789,6 +1789,7 @@ dropdb(const char *dbname, bool missing_ok, bool force) * Tell the cumulative stats system to forget it immediately, too. */ pgstat_drop_database(db_id); + pgstat_drop_vacuum_database(db_id); /* * Except for the deletion of the catalog row, subsequent actions are not diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 33a33bf6b1cf..ffb7e1eef4c7 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -115,6 +115,9 @@ pg_atomic_uint32 *VacuumSharedCostBalance = NULL; pg_atomic_uint32 *VacuumActiveNWorkers = NULL; int VacuumCostBalanceLocal = 0; +/* Cumulative storage to report total vacuum delay time. */ +double VacuumDelayTime = 0; /* msec. */ + /* non-export function prototypes */ static List *expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, int options); @@ -2514,6 +2517,7 @@ vacuum_delay_point(bool is_analyze) exit(1); VacuumCostBalance = 0; + VacuumDelayTime += msec; /* * Balance and update limit values for autovacuum workers. We must do diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 2b9d548cdeb1..9401e46d7552 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -868,6 +868,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + LVExtStatCountersIdx extVacCounters; + PgStat_VacuumRelationCounts extVacReport; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -876,6 +878,9 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); + /* Set initial statistics values to gather vacuum statistics for the index */ + extvac_stats_start_idx(indrel, &(indstats->istat), &extVacCounters); + ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -904,6 +909,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + /* Make extended vacuum stats report for index */ + extvac_stats_end_idx(indrel, istat_res, &extVacCounters, &extVacReport); + pgstat_report_tab_vacuum_extstats(RelationGetRelid(indrel), indrel->rd_rel->relisshared, + &extVacReport); + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they @@ -1054,6 +1064,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) /* Set cost-based vacuum delay */ VacuumUpdateCosts(); VacuumCostBalance = 0; + VacuumDelayTime = 0; VacuumCostBalanceLocal = 0; VacuumSharedCostBalance = &(shared->cost_balance); VacuumActiveNWorkers = &(shared->active_nworkers); diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile index 9c2443e1ecd3..183f7514d2dc 100644 --- a/src/backend/utils/activity/Makefile +++ b/src/backend/utils/activity/Makefile @@ -27,6 +27,7 @@ OBJS = \ pgstat_function.o \ pgstat_io.o \ pgstat_relation.o \ + pgstat_vacuum.o \ pgstat_replslot.o \ pgstat_shmem.o \ pgstat_slru.o \ diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 8b57845e8709..ca764a3a214d 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -190,7 +190,7 @@ static void pgstat_reset_after_failure(void); static bool pgstat_flush_pending_entries(bool nowait); static void pgstat_prep_snapshot(void); -static void pgstat_build_snapshot(void); +static void pgstat_build_snapshot(PgStat_Kind statKind); static void pgstat_build_snapshot_fixed(PgStat_Kind kind); static inline bool pgstat_is_kind_valid(PgStat_Kind kind); @@ -203,7 +203,7 @@ static inline bool pgstat_is_kind_valid(PgStat_Kind kind); bool pgstat_track_counts = false; int pgstat_fetch_consistency = PGSTAT_FETCH_CONSISTENCY_CACHE; - +bool pgstat_track_vacuum_statistics = false; /* ---------- * state shared with pgstat_*.c @@ -260,7 +260,6 @@ static bool pgstat_is_initialized = false; static bool pgstat_is_shutdown = false; #endif - /* * The different kinds of built-in statistics. * @@ -479,6 +478,34 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .reset_all_cb = pgstat_wal_reset_all_cb, .snapshot_cb = pgstat_wal_snapshot_cb, }, + [PGSTAT_KIND_VACUUM_DB] = { + .name = "vacuum statistics", + + .fixed_amount = false, + .write_to_file = true, + /* so pg_stat_database entries can be seen in all databases */ + .accessed_across_databases = true, + + .shared_size = sizeof(PgStatShared_VacuumDB), + .shared_data_off = offsetof(PgStatShared_VacuumDB, stats), + .shared_data_len = sizeof(((PgStatShared_VacuumDB *) 0)->stats), + .pending_size = sizeof(PgStat_VacuumDBCounts), + + .flush_pending_cb = pgstat_vacuum_db_flush_cb, + }, + [PGSTAT_KIND_VACUUM_RELATION] = { + .name = "vacuum statistics", + + .fixed_amount = false, + .write_to_file = true, + + .shared_size = sizeof(PgStatShared_VacuumRelation), + .shared_data_off = offsetof(PgStatShared_VacuumRelation, stats), + .shared_data_len = sizeof(((PgStatShared_VacuumRelation *) 0)->stats), + .pending_size = sizeof(PgStat_RelationVacuumPending), + + .flush_pending_cb = pgstat_vacuum_relation_flush_cb + }, }; /* @@ -897,7 +924,6 @@ pgstat_reset_of_kind(PgStat_Kind kind) pgstat_reset_entries_of_kind(kind, ts); } - /* ------------------------------------------------------------ * Fetching of stats * ------------------------------------------------------------ @@ -966,7 +992,7 @@ pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, uint64 objid) /* if we need to build a full snapshot, do so */ if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_SNAPSHOT) - pgstat_build_snapshot(); + pgstat_build_snapshot(PGSTAT_KIND_INVALID); /* if caching is desired, look up in cache */ if (pgstat_fetch_consistency > PGSTAT_FETCH_CONSISTENCY_NONE) @@ -1082,7 +1108,7 @@ pgstat_snapshot_fixed(PgStat_Kind kind) pgstat_clear_snapshot(); if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_SNAPSHOT) - pgstat_build_snapshot(); + pgstat_build_snapshot(PGSTAT_KIND_INVALID); else pgstat_build_snapshot_fixed(kind); @@ -1133,7 +1159,7 @@ pgstat_prep_snapshot(void) } static void -pgstat_build_snapshot(void) +pgstat_build_snapshot(PgStat_Kind statKind) { dshash_seq_status hstat; PgStatShared_HashEntry *p; @@ -1178,6 +1204,10 @@ pgstat_build_snapshot(void) if (p->dropped) continue; + if (statKind != PGSTAT_KIND_INVALID && statKind != p->key.kind) + /* Load stat of specific type, if defined */ + continue; + Assert(pg_atomic_read_u32(&p->refcount) > 0); stats_data = dsa_get_address(pgStatLocal.dsa, p->body); diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index b31f20d41bcc..80e6c7c229a6 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -46,6 +46,15 @@ pgstat_drop_database(Oid databaseid) pgstat_drop_transactional(PGSTAT_KIND_DATABASE, databaseid, InvalidOid); } +/* + * Remove entry for the database being dropped. + */ +void +pgstat_drop_vacuum_database(Oid databaseid) +{ + pgstat_drop_transactional(PGSTAT_KIND_VACUUM_DB, databaseid, InvalidOid); +} + /* * Called from autovacuum.c to report startup of an autovacuum process. * We are called before InitPostgres is done, so can't rely on MyDatabaseId; diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 28587e2916b1..acc8f0b8a52f 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -881,6 +881,9 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) tabentry->blocks_fetched += lstats->counts.blocks_fetched; tabentry->blocks_hit += lstats->counts.blocks_hit; + tabentry->rev_all_frozen_pages += lstats->counts.rev_all_frozen_pages; + tabentry->rev_all_visible_pages += lstats->counts.rev_all_visible_pages; + /* Clamp live_tuples in case of negative delta_live_tuples */ tabentry->live_tuples = Max(tabentry->live_tuples, 0); /* Likewise for dead_tuples */ @@ -901,6 +904,12 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) return true; } +void +pgstat_vacuum_relation_delete_pending_cb(Oid relid) +{ + pgstat_drop_transactional(PGSTAT_KIND_VACUUM_RELATION, relid, InvalidOid); +} + void pgstat_relation_delete_pending_cb(PgStat_EntryRef *entry_ref) { @@ -1003,4 +1012,4 @@ restore_truncdrop_counters(PgStat_TableXactStatus *trans) trans->tuples_updated = trans->updated_pre_truncdrop; trans->tuples_deleted = trans->deleted_pre_truncdrop; } -} +} \ No newline at end of file diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 97af7c6554ff..423a256c83a0 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -106,6 +106,12 @@ PG_STAT_GET_RELENTRY_INT64(tuples_updated) /* pg_stat_get_vacuum_count */ PG_STAT_GET_RELENTRY_INT64(vacuum_count) +/* pg_stat_get_rev_frozen_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_frozen_pages) + +/* pg_stat_get_rev_all_visible_pages */ +PG_STAT_GET_RELENTRY_INT64(rev_all_visible_pages) + #define PG_STAT_GET_RELENTRY_FLOAT8(stat) \ Datum \ CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ @@ -2258,3 +2264,215 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objid)); } + + +/* + * Get the vacuum statistics for the heap tables. + */ +Datum +pg_stat_get_vacuum_tables(PG_FUNCTION_ARGS) +{ + #define PG_STAT_GET_VACUUM_TABLES_STATS_COLS 26 + + Oid relid = PG_GETARG_OID(0); + PgStat_VacuumRelationCounts *extvacuum; + PgStat_VacuumRelationCounts *pending; + TupleDesc tupdesc; + Datum values[PG_STAT_GET_VACUUM_TABLES_STATS_COLS] = {0}; + bool nulls[PG_STAT_GET_VACUUM_TABLES_STATS_COLS] = {0}; + char buf[256]; + int i = 0; + PgStat_VacuumRelationCounts allzero; + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + pending = pgstat_fetch_stat_vacuum_tabentry(relid, MyDatabaseId); + + if (pending == NULL) + { + /* If the subscription is not found, initialise its stats */ + memset(&allzero, 0, sizeof(PgStat_VacuumRelationCounts)); + extvacuum = &allzero; + } + else + extvacuum = pending; + + i = 0; + + values[i++] = ObjectIdGetDatum(relid); + + values[i++] = Int64GetDatum(extvacuum->total_blks_read); + values[i++] = Int64GetDatum(extvacuum->total_blks_hit); + values[i++] = Int64GetDatum(extvacuum->total_blks_dirtied); + values[i++] = Int64GetDatum(extvacuum->total_blks_written); + + values[i++] = Int64GetDatum(extvacuum->blks_fetched - + extvacuum->blks_hit); + values[i++] = Int64GetDatum(extvacuum->blks_hit); + + values[i++] = Int64GetDatum(extvacuum->table.pages_scanned); + values[i++] = Int64GetDatum(extvacuum->table.pages_removed); + values[i++] = Int64GetDatum(extvacuum->table.vm_new_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->table.vm_new_visible_pages); + values[i++] = Int64GetDatum(extvacuum->table.vm_new_visible_frozen_pages); + values[i++] = Int64GetDatum(extvacuum->table.missed_dead_pages); + values[i++] = Int64GetDatum(extvacuum->tuples_deleted); + values[i++] = Int64GetDatum(extvacuum->table.tuples_frozen); + values[i++] = Int64GetDatum(extvacuum->table.recently_dead_tuples); + values[i++] = Int64GetDatum(extvacuum->table.missed_dead_tuples); + + values[i++] = Int32GetDatum(extvacuum->wraparound_failsafe_count); + values[i++] = Int64GetDatum(extvacuum->table.index_vacuum_count); + + values[i++] = Int64GetDatum(extvacuum->wal_records); + values[i++] = Int64GetDatum(extvacuum->wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, extvacuum->wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(extvacuum->blk_read_time); + values[i++] = Float8GetDatum(extvacuum->blk_write_time); + values[i++] = Float8GetDatum(extvacuum->delay_time); + values[i++] = Float8GetDatum(extvacuum->total_time); + + Assert(i == PG_STAT_GET_VACUUM_TABLES_STATS_COLS); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + +/* + * Get the vacuum statistics for the heap tables. + */ +Datum +pg_stat_get_vacuum_indexes(PG_FUNCTION_ARGS) +{ + #define PG_STAT_GET_VACUUM_INDEX_STATS_COLS 16 + + Oid relid = PG_GETARG_OID(0); + PgStat_VacuumRelationCounts *extvacuum; + PgStat_VacuumRelationCounts *pending; + TupleDesc tupdesc; + Datum values[PG_STAT_GET_VACUUM_INDEX_STATS_COLS] = {0}; + bool nulls[PG_STAT_GET_VACUUM_INDEX_STATS_COLS] = {0}; + char buf[256]; + int i = 0; + PgStat_VacuumRelationCounts allzero; + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + pending = pgstat_fetch_stat_vacuum_tabentry(relid, MyDatabaseId); + + if (pending == NULL) + { + /* If the subscription is not found, initialise its stats */ + memset(&allzero, 0, sizeof(PgStat_VacuumRelationCounts)); + extvacuum = &allzero; + } + else + extvacuum = pending; + + i = 0; + + values[i++] = ObjectIdGetDatum(relid); + + values[i++] = Int64GetDatum(extvacuum->total_blks_read); + values[i++] = Int64GetDatum(extvacuum->total_blks_hit); + values[i++] = Int64GetDatum(extvacuum->total_blks_dirtied); + values[i++] = Int64GetDatum(extvacuum->total_blks_written); + + values[i++] = Int64GetDatum(extvacuum->blks_fetched - + extvacuum->blks_hit); + values[i++] = Int64GetDatum(extvacuum->blks_hit); + + values[i++] = Int64GetDatum(extvacuum->index.pages_deleted); + values[i++] = Int64GetDatum(extvacuum->tuples_deleted); + + values[i++] = Int64GetDatum(extvacuum->wal_records); + values[i++] = Int64GetDatum(extvacuum->wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, extvacuum->wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(extvacuum->blk_read_time); + values[i++] = Float8GetDatum(extvacuum->blk_write_time); + values[i++] = Float8GetDatum(extvacuum->delay_time); + values[i++] = Float8GetDatum(extvacuum->total_time); + + Assert(i == PG_STAT_GET_VACUUM_INDEX_STATS_COLS); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + +Datum +pg_stat_get_vacuum_database(PG_FUNCTION_ARGS) +{ + #define PG_STAT_GET_VACUUM_DATABASE_STATS_COLS 14 + + Oid dbid = PG_GETARG_OID(0); + PgStat_VacuumDBCounts *extvacuum; + PgStat_VacuumDBCounts *pending; + TupleDesc tupdesc; + Datum values[PG_STAT_GET_VACUUM_DATABASE_STATS_COLS] = {0}; + bool nulls[PG_STAT_GET_VACUUM_DATABASE_STATS_COLS] = {0}; + char buf[256]; + int i = 0; + PgStat_VacuumDBCounts allzero; + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + pending = pgstat_fetch_stat_vacuum_dbentry(dbid); + + if (pending == NULL) + { + /* If the subscription is not found, initialise its stats */ + memset(&allzero, 0, sizeof(PgStat_VacuumRelationCounts)); + extvacuum = &allzero; + } + else + extvacuum = pending; + + values[i++] = ObjectIdGetDatum(dbid); + + values[i++] = Int64GetDatum(extvacuum->total_blks_read); + values[i++] = Int64GetDatum(extvacuum->total_blks_hit); + values[i++] = Int64GetDatum(extvacuum->total_blks_dirtied); + values[i++] = Int64GetDatum(extvacuum->total_blks_written); + + values[i++] = Int64GetDatum(extvacuum->wal_records); + values[i++] = Int64GetDatum(extvacuum->wal_fpi); + + /* Convert to numeric, like pg_stat_statements */ + snprintf(buf, sizeof buf, UINT64_FORMAT, extvacuum->wal_bytes); + values[i++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + + values[i++] = Float8GetDatum(extvacuum->blk_read_time); + values[i++] = Float8GetDatum(extvacuum->blk_write_time); + values[i++] = Float8GetDatum(extvacuum->delay_time); + values[i++] = Float8GetDatum(extvacuum->total_time); + values[i++] = Int32GetDatum(extvacuum->wraparound_failsafe_count); + values[i++] = Int32GetDatum(extvacuum->errors); + + Assert(i == PG_STAT_GET_VACUUM_DATABASE_STATS_COLS); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} \ No newline at end of file diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 47af743990fe..8c9e8fb18e14 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -1624,6 +1624,19 @@ getinternalerrposition(void) return edata->internalpos; } +/* + * Return elevel of errors + */ +int +geterrelevel(void) +{ + ErrorData *edata = &errordata[errordata_stack_depth]; + + /* we don't bother incrementing recursion_depth */ + CHECK_STACK_DEPTH(); + + return edata->elevel; +} /* * Functions to allow construction of error message strings separately from diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 2f8cbd867599..a24dec63f3ac 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1508,6 +1508,15 @@ struct config_bool ConfigureNamesBool[] = false, NULL, NULL, NULL }, + { + {"track_vacuum_statistics", PGC_SUSET, STATS_CUMULATIVE, + gettext_noop("Collects vacuum statistics for relations."), + NULL + }, + &pgstat_track_vacuum_statistics, + false, + NULL, NULL, NULL + }, { {"track_wal_io_timing", PGC_SUSET, STATS_CUMULATIVE, gettext_noop("Collects timing statistics for WAL I/O activity."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 34826d01380b..b4971a9dffeb 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -665,6 +665,7 @@ #track_wal_io_timing = off #track_functions = none # none, pl, all #stats_fetch_consistency = cache # cache, none, snapshot +#track_vacuum_statistics = off # - Monitoring - diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 62beb71da288..4710bf997c42 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12566,4 +12566,40 @@ proargnames => '{pid,io_id,io_generation,state,operation,off,length,target,handle_data_len,raw_result,result,target_desc,f_sync,f_localmem,f_buffered}', prosrc => 'pg_get_aios' }, +{ oid => '8001', + descr => 'pg_stat_get_vacuum_tables returns vacuum stats values for table', + proname => 'pg_stat_get_vacuum_tables', prorows => 1000, provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int4,int8,int8,int8,numeric,float8,float8,float8,float8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_scanned,pages_removed,vm_new_frozen_pages,vm_new_visible_pages,vm_new_visible_frozen_pages,missed_dead_pages,tuples_deleted,tuples_frozen,recently_dead_tuples,missed_dead_tuples,wraparound_failsafe,index_vacuum_count,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,total_time}', + prosrc => 'pg_stat_get_vacuum_tables' }, + + { oid => '8002', descr => 'statistics: number of times the all-visible pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_visible_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_visible_pages' }, + { oid => '8003', descr => 'statistics: number of times the all-frozen pages in the visibility map was removed for pages of table', + proname => 'pg_stat_get_rev_all_frozen_pages', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_rev_all_frozen_pages' }, +{ oid => '8004', + descr => 'pg_stat_get_vacuum_indexes returns vacuum stats values for index', + proname => 'pg_stat_get_vacuum_indexes', prorows => 1000, provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{reloid,relid,total_blks_read,total_blks_hit,total_blks_dirtied,total_blks_written,rel_blks_read,rel_blks_hit,pages_deleted,tuples_deleted,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,total_time}', + prosrc => 'pg_stat_get_vacuum_indexes' }, +{ oid => '8005', + descr => 'pg_stat_get_vacuum_database returns vacuum stats values for database', + proname => 'pg_stat_get_vacuum_database', prorows => 1000, provolatile => 's', prorettype => 'record',proisstrict => 'f', + proretset => 't', + proargtypes => 'oid', + proallargtypes => '{oid,oid,int8,int8,int8,int8,int8,int8,numeric,float8,float8,float8,float8,int4,int4}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{dbid,dboid,db_blks_read,db_blks_hit,total_blks_dirtied,total_blks_written,wal_records,wal_fpi,wal_bytes,blk_read_time,blk_write_time,delay_time,total_time,wraparound_failsafe,errors}', + prosrc => 'pg_stat_get_vacuum_database' }, ] diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index bc37a80dc74f..f895151ca090 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -25,6 +25,7 @@ #include "storage/buf.h" #include "storage/lock.h" #include "utils/relcache.h" +#include "pgstat.h" /* * Flags for amparallelvacuumoptions to control the participation of bulkdelete @@ -295,6 +296,26 @@ typedef struct VacDeadItemsInfo int64 num_items; /* current # of entries */ } VacDeadItemsInfo; +/* + * Counters and usage data for extended stats tracking. + */ +typedef struct LVExtStatCounters +{ + TimestampTz starttime; + WalUsage walusage; + BufferUsage bufusage; + double VacuumDelayTime; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; +} LVExtStatCounters; + +typedef struct LVExtStatCountersIdx +{ + LVExtStatCounters common; + int64 pages_deleted; + int64 tuples_removed; +} LVExtStatCountersIdx; + /* GUC parameters */ extern PGDLLIMPORT int default_statistics_target; /* PGDLLIMPORT for PostGIS */ extern PGDLLIMPORT int vacuum_freeze_min_age; @@ -327,6 +348,7 @@ extern PGDLLIMPORT double vacuum_max_eager_freeze_failure_rate; extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +extern PGDLLIMPORT double VacuumDelayTime; extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; @@ -407,4 +429,8 @@ extern double anl_random_fract(void); extern double anl_init_selection_state(int n); extern double anl_get_next_S(double t, int n, double *stateptr); +extern void extvac_stats_start_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters); +extern void extvac_stats_end_idx(Relation rel, IndexBulkDeleteResult *stats, + LVExtStatCountersIdx *counters, PgStat_VacuumRelationCounts *report); #endif /* VACUUM_H */ diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 378f2f2c2ba2..f57a96d3aa26 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -111,6 +111,15 @@ typedef struct PgStat_BackendSubEntry PgStat_Counter conflict_count[CONFLICT_NUM_TYPES]; } PgStat_BackendSubEntry; +/* Type of ExtVacReport */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_TABLE = 1, + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, +} ExtVacReportType; + /* ---------- * PgStat_TableCounts The actual per-table counts kept by a backend * @@ -153,8 +162,122 @@ typedef struct PgStat_TableCounts PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; } PgStat_TableCounts; +/* ---------- + * + * PgStat_VacuumRelationCounts + * + * Additional statistics of vacuum processing over a relation. + * pages_removed is the amount by which the physically shrank, + * if any (ie the change in its total size on disk) + * pages_deleted refer to free space within the index file + * ---------- + */ +typedef struct PgStat_VacuumRelationCounts +{ + /* number of blocks missed, hit, dirtied and written during a vacuum of specific relation */ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + + /* blocks missed and hit for just the heap during a vacuum of specific relation */ + int64 blks_fetched; + int64 blks_hit; + + /* Vacuum WAL usage stats */ + int64 wal_records; /* wal usage: number of WAL records */ + int64 wal_fpi; /* wal usage: number of WAL full page images produced */ + uint64 wal_bytes; /* wal usage: size of WAL records produced */ + + /* Time stats. */ + double blk_read_time; /* time spent reading pages, in msec */ + double blk_write_time; /* time spent writing pages, in msec */ + double delay_time; /* how long vacuum slept in vacuum delay point, in msec */ + double total_time; /* total time of a vacuum operation, in msec */ + + int64 tuples_deleted; /* tuples deleted by vacuum */ + + int32 wraparound_failsafe_count; /* the number of times to prevent wraparound problem */ + + ExtVacReportType type; /* heap, index, etc. */ + + /* ---------- + * + * There are separate metrics of statistic for tables and indexes, + * which collect during vacuum. + * The union operator allows to combine these statistics + * so that each metric is assigned to a specific class of collected statistics. + * Such a combined structure was called per_type_stats. + * The name of the structure itself is not used anywhere, + * it exists only for understanding the code. + * ---------- + */ + union + { + struct + { + int64 pages_scanned; /* heap pages examined (not skipped by VM) */ + int64 pages_removed; /* heap pages removed by vacuum "truncation" */ + int64 pages_frozen; /* pages marked in VM as frozen */ + int64 pages_all_visible; /* pages marked in VM as all-visible */ + int64 tuples_frozen; /* tuples frozen up by vacuum */ + int64 recently_dead_tuples; /* deleted tuples that are still visible to some transaction */ + int64 vm_new_frozen_pages; /* pages marked in VM as frozen */ + int64 vm_new_visible_pages; /* pages marked in VM as all-visible */ + int64 vm_new_visible_frozen_pages; /* pages marked in VM as all-visible and frozen */ + int64 missed_dead_tuples; /* tuples not pruned by vacuum due to failure to get a cleanup lock */ + int64 missed_dead_pages; /* pages with missed dead tuples */ + int64 index_vacuum_count; /* number of index vacuumings */ + } table; + struct + { + int64 pages_deleted; /* number of pages deleted by vacuum */ + } index; + } /* per_type_stats */; +} PgStat_VacuumRelationCounts; + +typedef struct PgStat_VacuumRelationStatus +{ + Oid id; /* table's OID */ + bool shared; /* is it a shared catalog? */ + PgStat_VacuumRelationCounts counts; /* event counts to be sent */ +} PgStat_VacuumRelationStatus; + +typedef struct PgStat_VacuumDBCounts +{ + Oid dbjid; + /* number of blocks missed, hit, dirtied and written during a vacuum of specific relation */ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + + /* blocks missed and hit for just the heap during a vacuum of specific relation */ + int64 blks_fetched; + int64 blks_hit; + + /* Vacuum WAL usage stats */ + int64 wal_records; /* wal usage: number of WAL records */ + int64 wal_fpi; /* wal usage: number of WAL full page images produced */ + uint64 wal_bytes; /* wal usage: size of WAL records produced */ + + /* Time stats. */ + double blk_read_time; /* time spent reading pages, in msec */ + double blk_write_time; /* time spent writing pages, in msec */ + double delay_time; /* how long vacuum slept in vacuum delay point, in msec */ + double total_time; /* total time of a vacuum operation, in msec */ + + int64 tuples_deleted; /* tuples deleted by vacuum */ + + int32 errors; + int32 wraparound_failsafe_count; /* the number of times to prevent wraparound problem */ +} PgStat_VacuumDBCounts; + /* ---------- * PgStat_TableStatus Per-table status within a backend * @@ -179,6 +302,12 @@ typedef struct PgStat_TableStatus Relation relation; /* rel that is using this entry */ } PgStat_TableStatus; +typedef struct PgStat_RelationVacuumPending +{ + Oid id; /* table's OID */ + PgStat_VacuumRelationCounts counts; /* event counts to be sent */ +} PgStat_RelationVacuumPending; + /* ---------- * PgStat_TableXactStatus Per-table, per-subtransaction status * ---------- @@ -211,7 +340,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB7 +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB8 typedef struct PgStat_ArchiverStats { @@ -453,6 +582,9 @@ typedef struct PgStat_StatTabEntry PgStat_Counter total_autovacuum_time; PgStat_Counter total_analyze_time; PgStat_Counter total_autoanalyze_time; + + PgStat_Counter rev_all_visible_pages; + PgStat_Counter rev_all_frozen_pages; } PgStat_StatTabEntry; /* ------ @@ -664,6 +796,7 @@ extern void pgstat_report_vacuum(Oid tableoid, bool shared, extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter, TimestampTz starttime); +extern void pgstat_report_vacuum_error(); /* * If stats are enabled, but pending data hasn't been prepared yet, call @@ -711,6 +844,17 @@ extern void pgstat_report_analyze(Relation rel, if (pgstat_should_count_relation(rel)) \ (rel)->pgstat_info->counts.blocks_hit++; \ } while (0) +/* accumulate unfrozen all-visible and all-frozen pages */ +#define pgstat_count_vm_rev_all_visible(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_visible_pages++; \ + } while (0) +#define pgstat_count_vm_rev_all_frozen(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + (rel)->pgstat_info->counts.rev_all_frozen_pages++; \ + } while (0) extern void pgstat_count_heap_insert(Relation rel, PgStat_Counter n); extern void pgstat_count_heap_update(Relation rel, bool hot, bool newpage); @@ -783,6 +927,15 @@ extern int pgstat_get_transactional_drops(bool isCommit, struct xl_xact_stats_it extern void pgstat_execute_transactional_drops(int ndrops, struct xl_xact_stats_item *items, bool is_redo); +extern void pgstat_drop_vacuum_database(Oid databaseid); +extern void pgstat_vacuum_relation_delete_pending_cb(Oid relid); +extern void +pgstat_report_tab_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts *params); +extern PgStat_RelationVacuumPending * find_vacuum_relation_entry(Oid relid); +extern PgStat_VacuumDBCounts *pgstat_prep_vacuum_database_pending(Oid dboid); +extern PgStat_VacuumRelationCounts *pgstat_fetch_stat_vacuum_tabentry(Oid relid, Oid dbid); +PgStat_VacuumDBCounts *pgstat_fetch_stat_vacuum_dbentry(Oid dbid); /* * Functions in pgstat_wal.c */ @@ -799,6 +952,7 @@ extern PgStat_WalStats *pgstat_fetch_stat_wal(void); extern PGDLLIMPORT bool pgstat_track_counts; extern PGDLLIMPORT int pgstat_track_functions; extern PGDLLIMPORT int pgstat_fetch_consistency; +extern PGDLLIMPORT bool pgstat_track_vacuum_statistics; /* diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 5eac0e16970c..6a30a4db47da 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -230,6 +230,7 @@ extern int geterrcode(void); extern int geterrposition(void); extern int getinternalerrposition(void); +extern int geterrelevel(void); /*---------- * Old-style error reporting API: to be used in this way: diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index d5557e6e998c..140adbcdbd67 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -439,6 +439,18 @@ typedef struct PgStatShared_Relation PgStat_StatTabEntry stats; } PgStatShared_Relation; +typedef struct PgStatShared_VacuumDB +{ + PgStatShared_Common header; + PgStat_VacuumDBCounts stats; +} PgStatShared_VacuumDB; + +typedef struct PgStatShared_VacuumRelation +{ + PgStatShared_Common header; + PgStat_VacuumRelationCounts stats; +} PgStatShared_VacuumRelation; + typedef struct PgStatShared_Function { PgStatShared_Common header; @@ -607,6 +619,9 @@ extern PgStat_EntryRef *pgstat_fetch_pending_entry(PgStat_Kind kind, extern void *pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, uint64 objid); extern void pgstat_snapshot_fixed(PgStat_Kind kind); +bool pgstat_vacuum_db_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern bool pgstat_vacuum_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); + /* * Functions in pgstat_archiver.c diff --git a/src/include/utils/pgstat_kind.h b/src/include/utils/pgstat_kind.h index f44169fd5a3c..454661f9d6a8 100644 --- a/src/include/utils/pgstat_kind.h +++ b/src/include/utils/pgstat_kind.h @@ -38,9 +38,11 @@ #define PGSTAT_KIND_IO 10 #define PGSTAT_KIND_SLRU 11 #define PGSTAT_KIND_WAL 12 +#define PGSTAT_KIND_VACUUM_DB 13 +#define PGSTAT_KIND_VACUUM_RELATION 14 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE -#define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL +#define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_VACUUM_RELATION #define PGSTAT_KIND_BUILTIN_SIZE (PGSTAT_KIND_BUILTIN_MAX + 1) /* Custom stats kinds */ diff --git a/src/test/isolation/expected/vacuum-extending-in-repetable-read.out b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out new file mode 100644 index 000000000000..6d9604239120 --- /dev/null +++ b/src/test/isolation/expected/vacuum-extending-in-repetable-read.out @@ -0,0 +1,53 @@ +unused step name: s2_delete +Parsed test spec with 2 sessions + +starting permutation: s2_insert s2_print_vacuum_stats_table s1_begin_repeatable_read s2_update s2_insert_interrupt s2_vacuum s2_print_vacuum_stats_table s1_commit s2_checkpoint s2_vacuum s2_print_vacuum_stats_table +step s2_insert: INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 0| 0| 0| 0 +(1 row) + +step s1_begin_repeatable_read: + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + +count +----- + 100 +(1 row) + +step s2_update: UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; +step s2_insert_interrupt: INSERT INTO test_vacuum_stat_isolation values (1,1); +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 0| 600| 0| 0| 0 +(1 row) + +step s1_commit: COMMIT; +step s2_checkpoint: CHECKPOINT; +step s2_vacuum: VACUUM test_vacuum_stat_isolation; +step s2_print_vacuum_stats_table: + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; + +relname |tuples_deleted|recently_dead_tuples|missed_dead_tuples|missed_dead_pages|tuples_frozen +--------------------------+--------------+--------------------+------------------+-----------------+------------- +test_vacuum_stat_isolation| 300| 600| 0| 0| 303 +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index e3c669a29c7a..6faee3ad2c30 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -96,6 +96,7 @@ test: timeouts test: vacuum-concurrent-drop test: vacuum-conflict test: vacuum-skip-locked +test: vacuum-extending-in-repetable-read test: stats test: horizons test: predicate-hash diff --git a/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec new file mode 100644 index 000000000000..cfec31595801 --- /dev/null +++ b/src/test/isolation/specs/vacuum-extending-in-repetable-read.spec @@ -0,0 +1,59 @@ +# Test for checking recently_dead_tuples, tuples_deleted and frozen tuples in pg_stat_vacuum_tables. +# recently_dead_tuples values are counted when vacuum hasn't cleared tuples because they were deleted recently. +# recently_dead_tuples aren't increased after releasing lock compared with tuples_deleted, which increased +# by the value of the cleared tuples that the vacuum managed to clear. + +setup +{ + CREATE TABLE test_vacuum_stat_isolation(id int, ival int) WITH (autovacuum_enabled = off); + SET track_io_timing = on; + SET track_vacuum_statistics TO 'on'; +} + +teardown +{ + DROP TABLE test_vacuum_stat_isolation CASCADE; + RESET track_io_timing; + RESET track_vacuum_statistics; +} + +session s1 +setup { + SET track_vacuum_statistics TO 'on'; + } +step s1_begin_repeatable_read { + BEGIN transaction ISOLATION LEVEL REPEATABLE READ; + select count(ival) from test_vacuum_stat_isolation where id>900; + } +step s1_commit { COMMIT; } + +session s2 +setup { + SET track_vacuum_statistics TO 'on'; + } +step s2_insert { INSERT INTO test_vacuum_stat_isolation(id, ival) SELECT ival, ival%10 FROM generate_series(1,1000) As ival; } +step s2_update { UPDATE test_vacuum_stat_isolation SET ival = ival + 2 where id > 900; } +step s2_delete { DELETE FROM test_vacuum_stat_isolation where id > 900; } +step s2_insert_interrupt { INSERT INTO test_vacuum_stat_isolation values (1,1); } +step s2_vacuum { VACUUM test_vacuum_stat_isolation; } +step s2_checkpoint { CHECKPOINT; } +step s2_print_vacuum_stats_table +{ + SELECT + vt.relname, vt.tuples_deleted, vt.recently_dead_tuples, vt.missed_dead_tuples, vt.missed_dead_pages, vt.tuples_frozen + FROM pg_stat_vacuum_tables vt, pg_class c + WHERE vt.relname = 'test_vacuum_stat_isolation' AND vt.relid = c.oid; +} + +permutation + s2_insert + s2_print_vacuum_stats_table + s1_begin_repeatable_read + s2_update + s2_insert_interrupt + s2_vacuum + s2_print_vacuum_stats_table + s1_commit + s2_checkpoint + s2_vacuum + s2_print_vacuum_stats_table \ No newline at end of file diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 6cf828ca8d0d..f63f25f94d8e 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1829,7 +1829,9 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_total_vacuum_time(c.oid) AS total_vacuum_time, pg_stat_get_total_autovacuum_time(c.oid) AS total_autovacuum_time, pg_stat_get_total_analyze_time(c.oid) AS total_analyze_time, - pg_stat_get_total_autoanalyze_time(c.oid) AS total_autoanalyze_time + pg_stat_get_total_autoanalyze_time(c.oid) AS total_autoanalyze_time, + pg_stat_get_rev_all_frozen_pages(c.oid) AS rev_all_frozen_pages, + pg_stat_get_rev_all_visible_pages(c.oid) AS rev_all_visible_pages FROM ((pg_class c LEFT JOIN pg_index i ON ((c.oid = i.indrelid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) @@ -2222,7 +2224,9 @@ pg_stat_sys_tables| SELECT relid, total_vacuum_time, total_autovacuum_time, total_analyze_time, - total_autoanalyze_time + total_autoanalyze_time, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname = ANY (ARRAY['pg_catalog'::name, 'information_schema'::name])) OR (schemaname ~ '^pg_toast'::text)); pg_stat_user_functions| SELECT p.oid AS funcid, @@ -2274,9 +2278,82 @@ pg_stat_user_tables| SELECT relid, total_vacuum_time, total_autovacuum_time, total_analyze_time, - total_autoanalyze_time + total_autoanalyze_time, + rev_all_frozen_pages, + rev_all_visible_pages FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_database| SELECT db.oid AS dboid, + db.datname AS dbname, + stats.db_blks_read, + stats.db_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time, + stats.wraparound_failsafe, + stats.errors + FROM pg_database db, + LATERAL pg_stat_get_vacuum_database(db.oid) stats(dboid, db_blks_read, db_blks_hit, total_blks_dirtied, total_blks_written, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, total_time, wraparound_failsafe, errors); +pg_stat_vacuum_indexes| SELECT rel.oid AS relid, + ns.nspname AS schemaname, + rel.relname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.pages_deleted, + stats.tuples_deleted, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time + FROM (pg_class rel + JOIN pg_namespace ns ON ((ns.oid = rel.relnamespace))), + LATERAL pg_stat_get_vacuum_indexes(rel.oid) stats(relid, total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written, rel_blks_read, rel_blks_hit, pages_deleted, tuples_deleted, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, total_time) + WHERE (rel.relkind = 'i'::"char"); +pg_stat_vacuum_tables| SELECT ns.nspname AS schemaname, + rel.relname, + stats.relid, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.pages_scanned, + stats.pages_removed, + stats.vm_new_frozen_pages, + stats.vm_new_visible_pages, + stats.vm_new_visible_frozen_pages, + stats.missed_dead_pages, + stats.tuples_deleted, + stats.tuples_frozen, + stats.recently_dead_tuples, + stats.missed_dead_tuples, + stats.wraparound_failsafe, + stats.index_vacuum_count, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.delay_time, + stats.total_time + FROM (pg_class rel + JOIN pg_namespace ns ON ((ns.oid = rel.relnamespace))), + LATERAL pg_stat_get_vacuum_tables(rel.oid) stats(relid, total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written, rel_blks_read, rel_blks_hit, pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, wraparound_failsafe, index_vacuum_count, wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time, delay_time, total_time) + WHERE (rel.relkind = 'r'::"char"); pg_stat_wal| SELECT wal_records, wal_fpi, wal_bytes, diff --git a/src/test/regress/expected/vacuum_index_statistics.out b/src/test/regress/expected/vacuum_index_statistics.out new file mode 100644 index 000000000000..9e5d33342c9f --- /dev/null +++ b/src/test/regress/expected/vacuum_index_statistics.out @@ -0,0 +1,183 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on + track_counts +-------------- + on +(1 row) + +\set sample_size 10000 +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; +SHOW track_vacuum_statistics; -- must be off + track_vacuum_statistics +------------------------- + off +(1 row) + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +DELETE FROM vestat WHERE x % 2 = 0; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- Must be empty. +SELECT * +FROM pg_stat_vacuum_indexes vt +WHERE vt.relname = 'vestat'; + relid | schemaname | relname | total_blks_read | total_blks_hit | total_blks_dirtied | total_blks_written | rel_blks_read | rel_blks_hit | pages_deleted | tuples_deleted | wal_records | wal_fpi | wal_bytes | blk_read_time | blk_write_time | delay_time | total_time +-------+------------+---------+-----------------+----------------+--------------------+--------------------+---------------+--------------+---------------+----------------+-------------+---------+-----------+---------------+----------------+------------+------------ +(0 rows) + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; +SET track_vacuum_statistics TO 'on'; +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +\set sample_size 10000 +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int primary key) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +SELECT oid AS ioid from pg_class where relname = 'vestat_pkey' \gset +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,relpages,pages_deleted,tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | 30 | 0 | 0 +(1 row) + +SELECT relpages AS irp +FROM pg_class c +WHERE relname = 'vestat_pkey' \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted = 0 AS pages_deleted,tuples_deleted > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- Look into WAL records deltas. +SELECT wal_records > 0 AS diWR, wal_bytes > 0 AS diWB +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey'; + diwr | diwb +------+------ + t | t +(1 row) + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- pages_removed must be increased +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd > 0 AS pages_deleted,tuples_deleted-:itd > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr, wal_bytes-:iwb AS diwb, wal_fpi-:ifpi AS difpi +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- WAL advance should be detected. +SELECT :diwr > 0 AS diWR, :diwb > 0 AS diWB; + diwr | diwb +------+------ + t | t +(1 row) + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr2, wal_bytes-:iwb AS diwb2, wal_fpi-:ifpi AS difpi2 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +-- WAL and other statistics advance should not be detected. +SELECT :diwr2=0 AS diWR, :difpi2=0 AS iFPI, :diwb2=0 AS diWB; + diwr | ifpi | diwb +------+------+------ + t | t | t +(1 row) + +SELECT vt.relname,relpages-:irp < 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | t | t | t +(1 row) + +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:iwr AS diwr3, wal_bytes-:iwb AS diwb3, wal_fpi-:ifpi AS difpi3 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset +--There are nothing changed +SELECT :diwr3=0 AS diWR, :difpi3=0 AS iFPI, :diwb3=0 AS diWB; + diwr | ifpi | diwb +------+------+------ + t | t | t +(1 row) + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The pages_frozen, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + relname | relpages | pages_deleted | tuples_deleted +-------------+----------+---------------+---------------- + vestat_pkey | f | t | t +(1 row) + +DROP TABLE vestat; +RESET track_vacuum_statistics; diff --git a/src/test/regress/expected/vacuum_tables_and_db_statistics.out b/src/test/regress/expected/vacuum_tables_and_db_statistics.out new file mode 100644 index 000000000000..0300e7b62764 --- /dev/null +++ b/src/test/regress/expected/vacuum_tables_and_db_statistics.out @@ -0,0 +1,296 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +SHOW track_counts; -- must be on + track_counts +-------------- + on +(1 row) + +\set sample_size 10000 +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; +SHOW track_vacuum_statistics; -- must be off + track_vacuum_statistics +------------------------- + off +(1 row) + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +DELETE FROM vestat WHERE x % 2 = 0; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- Must be empty. +SELECT relname,total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written,rel_blks_read, rel_blks_hit, +pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, +tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, index_vacuum_count, +wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time,delay_time, total_time +FROM pg_stat_vacuum_tables vt +WHERE vt.relname = 'vestat'; + relname | total_blks_read | total_blks_hit | total_blks_dirtied | total_blks_written | rel_blks_read | rel_blks_hit | pages_scanned | pages_removed | vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | missed_dead_pages | tuples_deleted | tuples_frozen | recently_dead_tuples | missed_dead_tuples | index_vacuum_count | wal_records | wal_fpi | wal_bytes | blk_read_time | blk_write_time | delay_time | total_time +---------+-----------------+----------------+--------------------+--------------------+---------------+--------------+---------------+---------------+---------------------+----------------------+-----------------------------+-------------------+----------------+---------------+----------------------+--------------------+--------------------+-------------+---------+-----------+---------------+----------------+------------+------------ + vestat | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 +(1 row) + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; +CREATE DATABASE regression_statistic_vacuum_db; +CREATE DATABASE regression_statistic_vacuum_db1; +\c regression_statistic_vacuum_db; +SET track_vacuum_statistics TO on; +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +SELECT oid AS roid from pg_class where relname = 'vestat' \gset +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,vm_new_frozen_pages,tuples_deleted,relpages,pages_scanned,pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | 0 | 0 | 455 | 0 | 0 +(1 row) + +SELECT relpages AS rp +FROM pg_class c +WHERE relname = 'vestat' \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,vm_new_frozen_pages > 0 AS vm_new_frozen_pages,tuples_deleted > 0 AS tuples_deleted,relpages-:rp = 0 AS relpages,pages_scanned > 0 AS pages_scanned,pages_removed = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | f | t | t | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- Look into WAL records deltas. +SELECT wal_records > 0 AS dWR, wal_bytes > 0 AS dWB +FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; + dwr | dwb +-----+----- + t | t +(1 row) + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- pages_removed must be increased +SELECT vt.relname,vm_new_frozen_pages-:fp > 0 AS vm_new_frozen_pages,tuples_deleted-:td > 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps > 0 AS pages_scanned,pages_removed-:pr > 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | f | t | f | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr, wal_bytes-:hwb AS dwb, wal_fpi-:hfpi AS dfpi +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- WAL advance should be detected. +SELECT :dwr > 0 AS dWR, :dwb > 0 AS dWB; + dwr | dwb +-----+----- + t | t +(1 row) + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr2, wal_bytes-:hwb AS dwb2, wal_fpi-:hfpi AS dfpi2 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +-- WAL and other statistics advance should not be detected. +SELECT :dwr2=0 AS dWR, :dfpi2=0 AS dFPI, :dwb2=0 AS dWB; + dwr | dfpi | dwb +-----+------+----- + t | t | t +(1 row) + +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp < 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | t | t | f | t | t +(1 row) + +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps,pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:hwr AS dwr3, wal_bytes-:hwb AS dwb3, wal_fpi-:hfpi AS dfpi3 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset +--There are nothing changed +SELECT :dwr3>0 AS dWR, :dfpi3=0 AS dFPI, :dwb3>0 AS dWB; + dwr | dfpi | dwb +-----+------+----- + t | t | t +(1 row) + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The vm_new_frozen_pages, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + relname | vm_new_frozen_pages | tuples_deleted | relpages | pages_scanned | pages_removed +---------+---------------------+----------------+----------+---------------+--------------- + vestat | t | t | f | t | t +(1 row) + +DROP TABLE vestat CASCADE; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +-- must be empty +SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | rev_all_frozen_pages | rev_all_visible_pages | vm_new_visible_frozen_pages +---------------------+----------------------+----------------------+-----------------------+----------------------------- + 0 | 0 | 0 | 0 | 0 +(1 row) + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- backend defreezed pages +SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + f | t | f | t | t +(1 row) + +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv,vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset +UPDATE vestat SET x = x + 1001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + f | t | f | f | f +(1 row) + +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +-- vacuum freezed pages +SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + vm_new_frozen_pages | vm_new_visible_pages | vm_new_visible_frozen_pages | rev_all_frozen_pages | rev_all_visible_pages +---------------------+----------------------+-----------------------------+----------------------+----------------------- + t | t | t | t | t +(1 row) + +DROP TABLE vestat CASCADE; +-- Now check vacuum statistics for current database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = current_database(); + dbname | db_blks_hit | total_blks_dirtied | total_blks_written | wal_records | wal_fpi | wal_bytes | total_time +--------------------------------+-------------+--------------------+--------------------+-------------+---------+-----------+------------ + regression_statistic_vacuum_db | t | t | t | t | t | t | t +(1 row) + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush +-------------------------- + +(1 row) + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +UPDATE vestat SET x = 10001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; +\c regression_statistic_vacuum_db1; +SET track_vacuum_statistics TO on; +-- Now check vacuum statistics for postgres database from another database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = 'regression_statistic_vacuum_db'; + dbname | db_blks_hit | total_blks_dirtied | total_blks_written | wal_records | wal_fpi | wal_bytes | total_time +--------------------------------+-------------+--------------------+--------------------+-------------+---------+-----------+------------ + regression_statistic_vacuum_db | t | t | t | t | t | t | t +(1 row) + +\c regression_statistic_vacuum_db +SET track_vacuum_statistics TO on; +DROP TABLE vestat CASCADE; +\c regression_statistic_vacuum_db1; +SET track_vacuum_statistics TO on; +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_get_vacuum_tables(0) +WHERE oid = 0; -- must be 0 + count +------- + 0 +(1 row) + +\c postgres +DROP DATABASE regression_statistic_vacuum_db1; +DROP DATABASE regression_statistic_vacuum_db; +RESET track_vacuum_statistics; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index a424be2a6bf0..fa2489716ccd 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -140,3 +140,9 @@ test: fast_default # run tablespace test at the end because it drops the tablespace created during # setup that other tests may use. test: tablespace + +# ---------- +# Check vacuum statistics +# ---------- +test: vacuum_index_statistics +test: vacuum_tables_and_db_statistics \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_index_statistics.sql b/src/test/regress/sql/vacuum_index_statistics.sql new file mode 100644 index 000000000000..9b7e645187d3 --- /dev/null +++ b/src/test/regress/sql/vacuum_index_statistics.sql @@ -0,0 +1,151 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- +-- conditio sine qua non +SHOW track_counts; -- must be on + +\set sample_size 10000 + +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; + +SHOW track_vacuum_statistics; -- must be off + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +DELETE FROM vestat WHERE x % 2 = 0; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- Must be empty. +SELECT * +FROM pg_stat_vacuum_indexes vt +WHERE vt.relname = 'vestat'; + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; + +SET track_vacuum_statistics TO 'on'; + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +\set sample_size 10000 +SET vacuum_freeze_min_age = 0; +SET vacuum_freeze_table_age = 0; +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int primary key) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +SELECT oid AS ioid from pg_class where relname = 'vestat_pkey' \gset + +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,relpages,pages_deleted,tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT relpages AS irp +FROM pg_class c +WHERE relname = 'vestat_pkey' \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted = 0 AS pages_deleted,tuples_deleted > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- Look into WAL records deltas. +SELECT wal_records > 0 AS diWR, wal_bytes > 0 AS diWB +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey'; + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- pages_removed must be increased +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd > 0 AS pages_deleted,tuples_deleted-:itd > 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr, wal_bytes-:iwb AS diwb, wal_fpi-:ifpi AS difpi +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- WAL advance should be detected. +SELECT :diwr > 0 AS diWR, :diwb > 0 AS diWB; + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables +SELECT wal_records-:iwr AS diwr2, wal_bytes-:iwb AS diwb2, wal_fpi-:ifpi AS difpi2 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +-- WAL and other statistics advance should not be detected. +SELECT :diwr2=0 AS diWR, :difpi2=0 AS iFPI, :diwb2=0 AS diWB; + +SELECT vt.relname,relpages-:irp < 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; +SELECT vt.relname,relpages AS irp,pages_deleted AS ipd,tuples_deleted AS itd +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS iwr,wal_bytes AS iwb,wal_fpi AS ifpi FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP ON) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:iwr AS diwr3, wal_bytes-:iwb AS diwb3, wal_fpi-:ifpi AS difpi3 +FROM pg_stat_vacuum_indexes WHERE relname = 'vestat_pkey' \gset + +--There are nothing changed +SELECT :diwr3=0 AS diWR, :difpi3=0 AS iFPI, :diwb3=0 AS diWB; + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The pages_frozen, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,relpages-:irp = 0 AS relpages,pages_deleted-:ipd = 0 AS pages_deleted,tuples_deleted-:itd = 0 AS tuples_deleted +FROM pg_stat_vacuum_indexes vt, pg_class c +WHERE vt.relname = 'vestat_pkey' AND vt.relid = c.oid; + +DROP TABLE vestat; +RESET track_vacuum_statistics; diff --git a/src/test/regress/sql/vacuum_tables_and_db_statistics.sql b/src/test/regress/sql/vacuum_tables_and_db_statistics.sql new file mode 100644 index 000000000000..ca7dbde93877 --- /dev/null +++ b/src/test/regress/sql/vacuum_tables_and_db_statistics.sql @@ -0,0 +1,242 @@ +-- +-- Test cumulative vacuum stats system +-- +-- Check the wall statistics collected during vacuum operation: +-- number of frozen and visible pages set by vacuum; +-- number of frozen and visible pages removed by backend. +-- Statistic wal_fpi is not displayed in this test because its behavior is unstable. +-- + +SHOW track_counts; -- must be on +\set sample_size 10000 + +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; + +SHOW track_vacuum_statistics; -- must be off + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +DELETE FROM vestat WHERE x % 2 = 0; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- Must be empty. +SELECT relname,total_blks_read, total_blks_hit, total_blks_dirtied, total_blks_written,rel_blks_read, rel_blks_hit, +pages_scanned, pages_removed, vm_new_frozen_pages, vm_new_visible_pages, vm_new_visible_frozen_pages, missed_dead_pages, +tuples_deleted, tuples_frozen, recently_dead_tuples, missed_dead_tuples, index_vacuum_count, +wal_records, wal_fpi, wal_bytes, blk_read_time, blk_write_time,delay_time, total_time +FROM pg_stat_vacuum_tables vt +WHERE vt.relname = 'vestat'; + +RESET track_vacuum_statistics; +DROP TABLE vestat CASCADE; + +CREATE DATABASE regression_statistic_vacuum_db; +CREATE DATABASE regression_statistic_vacuum_db1; +\c regression_statistic_vacuum_db; +SET track_vacuum_statistics TO on; + +-- not enabled by default, but we want to test it... +SET track_functions TO 'all'; + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +--SET stats_fetch_consistency = snapshot; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +SELECT oid AS roid from pg_class where relname = 'vestat' \gset + +DELETE FROM vestat WHERE x % 2 = 0; +-- Before the first vacuum execution extended stats view is empty. +SELECT vt.relname,vm_new_frozen_pages,tuples_deleted,relpages,pages_scanned,pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT relpages AS rp +FROM pg_class c +WHERE relname = 'vestat' \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- The table and index extended vacuum statistics should show us that +-- vacuum frozed pages and clean up pages, but pages_removed stayed the same +-- because of not full table have cleaned up +SELECT vt.relname,vm_new_frozen_pages > 0 AS vm_new_frozen_pages,tuples_deleted > 0 AS tuples_deleted,relpages-:rp = 0 AS relpages,pages_scanned > 0 AS pages_scanned,pages_removed = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- Look into WAL records deltas. +SELECT wal_records > 0 AS dWR, wal_bytes > 0 AS dWB +FROM pg_stat_vacuum_tables WHERE relname = 'vestat'; + +DELETE FROM vestat;; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- pages_removed must be increased +SELECT vt.relname,vm_new_frozen_pages-:fp > 0 AS vm_new_frozen_pages,tuples_deleted-:td > 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps > 0 AS pages_scanned,pages_removed-:pr > 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps, pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr, wal_bytes-:hwb AS dwb, wal_fpi-:hfpi AS dfpi +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- WAL advance should be detected. +SELECT :dwr > 0 AS dWR, :dwb > 0 AS dWB; + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +DELETE FROM vestat WHERE x % 2 = 0; +-- VACUUM FULL doesn't report to stat collector. So, no any advancements of statistics +-- are detected here. +VACUUM FULL vestat; +-- It is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables +SELECT wal_records-:hwr AS dwr2, wal_bytes-:hwb AS dwb2, wal_fpi-:hfpi AS dfpi2 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +-- WAL and other statistics advance should not be detected. +SELECT :dwr2=0 AS dWR, :dfpi2=0 AS dFPI, :dwb2=0 AS dWB; + +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp < 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; +SELECT vm_new_frozen_pages AS fp,tuples_deleted AS td,relpages AS rp, pages_scanned AS ps,pages_removed AS pr +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid \gset + +-- Store WAL advances into variables +SELECT wal_records AS hwr,wal_bytes AS hwb,wal_fpi AS hfpi FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +DELETE FROM vestat; +TRUNCATE vestat; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128, INDEX_CLEANUP OFF) vestat; +-- it is necessary to check the wal statistics +CHECKPOINT; + +-- Store WAL advances into variables after removing all tuples from the table +SELECT wal_records-:hwr AS dwr3, wal_bytes-:hwb AS dwb3, wal_fpi-:hfpi AS dfpi3 +FROM pg_stat_vacuum_tables WHERE relname = 'vestat' \gset + +--There are nothing changed +SELECT :dwr3>0 AS dWR, :dfpi3=0 AS dFPI, :dwb3>0 AS dWB; + +-- +-- Now, the table and index is compressed into zero number of pages. Check it +-- in vacuum extended statistics. +-- The vm_new_frozen_pages, pages_scanned values shouldn't be changed +-- +SELECT vt.relname,vm_new_frozen_pages-:fp = 0 AS vm_new_frozen_pages,tuples_deleted-:td = 0 AS tuples_deleted,relpages -:rp = 0 AS relpages,pages_scanned-:ps = 0 AS pages_scanned,pages_removed-:pr = 0 AS pages_removed +FROM pg_stat_vacuum_tables vt, pg_class c +WHERE vt.relname = 'vestat' AND vt.relid = c.oid; + +DROP TABLE vestat CASCADE; +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); + +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; + +-- must be empty +SELECT vm_new_frozen_pages, vm_new_visible_pages, rev_all_frozen_pages,rev_all_visible_pages,vm_new_visible_frozen_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- backend defreezed pages +SELECT vm_new_frozen_pages > 0 AS vm_new_frozen_pages,vm_new_visible_pages > 0 AS vm_new_visible_pages,vm_new_visible_frozen_pages > 0 AS vm_new_visible_frozen_pages,rev_all_frozen_pages = 0 AS rev_all_frozen_pages,rev_all_visible_pages = 0 AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv,vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset + +UPDATE vestat SET x = x + 1001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +SELECT vm_new_frozen_pages > :pf AS vm_new_frozen_pages,vm_new_visible_pages > :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages > :pvf AS vm_new_visible_frozen_pages,rev_all_frozen_pages > :hafp AS rev_all_frozen_pages,rev_all_visible_pages > :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; +SELECT vm_new_frozen_pages AS pf, vm_new_visible_pages AS pv, vm_new_visible_frozen_pages AS pvf, rev_all_frozen_pages AS hafp,rev_all_visible_pages AS havp +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid \gset + +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +-- vacuum freezed pages +SELECT vm_new_frozen_pages = :pf AS vm_new_frozen_pages,vm_new_visible_pages = :pv AS vm_new_visible_pages,vm_new_visible_frozen_pages = :pvf AS vm_new_visible_frozen_pages, rev_all_frozen_pages = :hafp AS rev_all_frozen_pages,rev_all_visible_pages = :havp AS rev_all_visible_pages +FROM pg_stat_vacuum_tables, pg_stat_all_tables WHERE pg_stat_vacuum_tables.relname = 'vestat' and pg_stat_vacuum_tables.relid = pg_stat_all_tables.relid; + +DROP TABLE vestat CASCADE; + +-- Now check vacuum statistics for current database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = current_database(); + +-- ensure pending stats are flushed +SELECT pg_stat_force_next_flush(); + +CREATE TABLE vestat (x int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vestat SELECT x FROM generate_series(1,:sample_size) as x; +ANALYZE vestat; +UPDATE vestat SET x = 10001; +VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) vestat; + +\c regression_statistic_vacuum_db1; +SET track_vacuum_statistics TO on; + +-- Now check vacuum statistics for postgres database from another database +SELECT dbname, + db_blks_hit > 0 AS db_blks_hit, + total_blks_dirtied > 0 AS total_blks_dirtied, + total_blks_written > 0 AS total_blks_written, + wal_records > 0 AS wal_records, + wal_fpi > 0 AS wal_fpi, + wal_bytes > 0 AS wal_bytes, + total_time > 0 AS total_time +FROM +pg_stat_vacuum_database +WHERE dbname = 'regression_statistic_vacuum_db'; + +\c regression_statistic_vacuum_db +SET track_vacuum_statistics TO on; + +DROP TABLE vestat CASCADE; + +\c regression_statistic_vacuum_db1; +SET track_vacuum_statistics TO on; +SELECT count(*) +FROM pg_database d +CROSS JOIN pg_stat_get_vacuum_tables(0) +WHERE oid = 0; -- must be 0 + +\c postgres +DROP DATABASE regression_statistic_vacuum_db1; +DROP DATABASE regression_statistic_vacuum_db; +RESET track_vacuum_statistics; \ No newline at end of file