From f98a4ee8c6d03f79c7af1755610f03d70fb19ac2 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Wed, 13 Oct 2021 21:33:06 +0300 Subject: [PATCH 1/2] PostgreSQL-14 support --- .travis.yml | 13 ++ README.md | 10 +- patches/REL_14_STABLE-ptrack-core.diff | 309 +++++++++++++++++++++++++ patches/master-ptrack-core.diff | 309 +++++++++++++++++++++++++ 4 files changed, 636 insertions(+), 5 deletions(-) create mode 100644 patches/REL_14_STABLE-ptrack-core.diff create mode 100644 patches/master-ptrack-core.diff diff --git a/.travis.yml b/.travis.yml index b3698e1..c3edfdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,14 @@ notifications: # keep in sync with codecov.yml number of builds env: + - PG_VERSION=15 PG_BRANCH=master TEST_CASE=tap + - PG_VERSION=15 PG_BRANCH=master TEST_CASE=tap MODE=legacy + - PG_VERSION=15 PG_BRANCH=master TEST_CASE=all + - PG_VERSION=15 PG_BRANCH=master TEST_CASE=all MODE=paranoia + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=tap + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=all + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=all @@ -33,3 +41,8 @@ env: - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia + +jobs: + allow_failures: + - if: env(PG_BRANCH) = master + diff --git a/README.md b/README.md index 0d6d232..b6b35fd 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files. -This extension is compatible with PostgreSQL [11](https://github.com/postgrespro/ptrack/blob/master/patches/REL_11_STABLE-ptrack-core.diff), [12](https://github.com/postgrespro/ptrack/blob/master/patches/REL_12_STABLE-ptrack-core.diff), [13](https://github.com/postgrespro/ptrack/blob/master/patches/REL_13_STABLE-ptrack-core.diff). +This extension is compatible with PostgreSQL [11](https://github.com/postgrespro/ptrack/blob/master/patches/REL_11_STABLE-ptrack-core.diff), [12](https://github.com/postgrespro/ptrack/blob/master/patches/REL_12_STABLE-ptrack-core.diff), [13](https://github.com/postgrespro/ptrack/blob/master/patches/REL_13_STABLE-ptrack-core.diff), [14](https://github.com/postgrespro/ptrack/blob/master/patches/REL_14_STABLE-ptrack-core.diff). ## Installation @@ -25,13 +25,13 @@ git clone https://github.com/postgrespro/ptrack.git 2) Get latest PostgreSQL sources: ```shell -git clone https://github.com/postgres/postgres.git -b REL_12_STABLE && cd postgres +git clone https://github.com/postgres/postgres.git -b REL_14_STABLE && cd postgres ``` 3) Apply PostgreSQL core patch: ```shell -git apply -3 ../ptrack/patches/REL_12_STABLE-ptrack-core.diff +git apply -3 ../ptrack/patches/REL_14_STABLE-ptrack-core.diff ``` 4) Compile and install PostgreSQL @@ -151,8 +151,8 @@ Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare) Everything is tested automatically with [travis-ci.com](https://travis-ci.com/postgrespro/ptrack) and [codecov.io](https://codecov.io/gh/postgrespro/ptrack), but you can also run tests locally via `Docker`: ```sh -export PG_VERSION=12 -export PG_BRANCH=REL_12_STABLE +export PG_VERSION=14 +export PG_BRANCH=REL_14_STABLE export TEST_CASE=all export MODE=paranoia diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff new file mode 100644 index 0000000..3491700 --- /dev/null +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -0,0 +1,309 @@ +commit a14ac459d71528c64df00c693e9c71ac70d3ba29 +Author: anastasia +Date: Mon Oct 19 14:53:06 2020 +0300 + + add ptrack 2.0 + +diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c +index 50ae1f16d0..721b926ad2 100644 +--- a/src/backend/replication/basebackup.c ++++ b/src/backend/replication/basebackup.c +@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ /* ++ * Skip all transient ptrack files, but do copy ptrack.map, since it may ++ * be successfully used immediately after backup. TODO: check, test? ++ */ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c +index 0cf598dd0c..c9c44a4ae7 100644 +--- a/src/backend/storage/file/copydir.c ++++ b/src/backend/storage/file/copydir.c +@@ -27,6 +27,8 @@ + #include "storage/copydir.h" + #include "storage/fd.h" + ++copydir_hook_type copydir_hook = NULL; ++ + /* + * copydir: copy a directory + * +@@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) + } + FreeDir(xldir); + ++ if (copydir_hook) ++ copydir_hook(todir); ++ + /* + * Be paranoid here and fsync all files to ensure the copy is really done. + * But if fsync is disabled, we're done. +diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c +index 0eacd461cd..c2ef404a1a 100644 +--- a/src/backend/storage/smgr/md.c ++++ b/src/backend/storage/smgr/md.c +@@ -87,6 +87,8 @@ typedef struct _MdfdVec + + static MemoryContext MdCxt; /* context for all MdfdVec objects */ + ++mdextend_hook_type mdextend_hook = NULL; ++mdwrite_hook_type mdwrite_hook = NULL; + + /* Populate a file tag describing an md.c segment file. */ + #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ +@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + register_dirty_segment(reln, forknum, v); + + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); ++ ++ if (mdextend_hook) ++ mdextend_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + + if (!skipFsync && !SmgrIsTemp(reln)) + register_dirty_segment(reln, forknum, v); ++ ++ if (mdwrite_hook) ++ mdwrite_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c +index 3ded2cdd71..3a596a59f7 100644 +--- a/src/backend/storage/sync/sync.c ++++ b/src/backend/storage/sync/sync.c +@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ + static CycleCtr sync_cycle_ctr = 0; + static CycleCtr checkpoint_cycle_ctr = 0; + ++ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; ++ + /* Intervals for calling AbsorbSyncRequests */ + #define FSYNCS_PER_ABSORB 10 + #define UNLINKS_PER_ABSORB 10 +@@ -420,6 +422,9 @@ ProcessSyncRequests(void) + CheckpointStats.ckpt_longest_sync = longest; + CheckpointStats.ckpt_agg_sync_time = total_elapsed; + ++ if (ProcessSyncRequests_hook) ++ ProcessSyncRequests_hook(); ++ + /* Flag successful completion of ProcessSyncRequests */ + sync_in_progress = false; + } +diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c +index 1683629ee3..d2fc154576 100644 +--- a/src/backend/utils/misc/guc.c ++++ b/src/backend/utils/misc/guc.c +@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; + static char *recovery_target_name_string; + static char *recovery_target_lsn_string; + +- + /* should be static, but commands/variable.c needs to get at this */ + char *role_string; + +diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c +index ffdc23945c..7ae95866ce 100644 +--- a/src/bin/pg_checksums/pg_checksums.c ++++ b/src/bin/pg_checksums/pg_checksums.c +@@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c +index 233441837f..cf7bd073bf 100644 +--- a/src/bin/pg_resetwal/pg_resetwal.c ++++ b/src/bin/pg_resetwal/pg_resetwal.c +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); + static void FindEndOfXLOG(void); + static void KillExistingXLOG(void); + static void KillExistingArchiveStatus(void); ++static void KillExistingPtrack(void); + static void WriteEmptyXLOG(void); + static void usage(void); + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) + RewriteControlFile(); + KillExistingXLOG(); + KillExistingArchiveStatus(); ++ KillExistingPtrack(); + WriteEmptyXLOG(); + + printf(_("Write-ahead log reset\n")); +@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) + } + } + ++/* ++ * Remove existing ptrack files ++ */ ++static void ++KillExistingPtrack(void) ++{ ++#define PTRACKDIR "global" ++ ++ DIR *xldir; ++ struct dirent *xlde; ++ char path[MAXPGPATH + sizeof(PTRACKDIR)]; ++ ++ xldir = opendir(PTRACKDIR); ++ if (xldir == NULL) ++ { ++ pg_log_error("could not open directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++ ++ while (errno = 0, (xlde = readdir(xldir)) != NULL) ++ { ++ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 || ++ strcmp(xlde->d_name, "ptrack.map") == 0 || ++ strcmp(xlde->d_name, "ptrack.map.tmp") == 0) ++ { ++ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name); ++ if (unlink(path) < 0) ++ { ++ pg_log_error("could not delete file \"%s\": %m", path); ++ exit(1); ++ } ++ } ++ } ++ ++ if (errno) ++ { ++ pg_log_error("could not read directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++ ++ if (closedir(xldir)) ++ { ++ pg_log_error("could not close directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++} ++ + + /* + * Write an empty XLOG file, containing only the checkpoint record +diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c +index fbb97b5cf1..6cd7f2ae3e 100644 +--- a/src/bin/pg_rewind/filemap.c ++++ b/src/bin/pg_rewind/filemap.c +@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h +index 72e3352398..5c2e016501 100644 +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -388,7 +388,7 @@ typedef enum ProcessingMode + NormalProcessing /* normal processing */ + } ProcessingMode; + +-extern ProcessingMode Mode; ++extern PGDLLIMPORT ProcessingMode Mode; + + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) + #define IsInitProcessingMode() (Mode == InitProcessing) +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 3c6f906683..a7355f7ad1 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif +diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h +index 5d28f59c1d..0d3f04d8af 100644 +--- a/src/include/storage/copydir.h ++++ b/src/include/storage/copydir.h +@@ -13,6 +13,9 @@ + #ifndef COPYDIR_H + #define COPYDIR_H + ++typedef void (*copydir_hook_type) (const char *path); ++extern PGDLLIMPORT copydir_hook_type copydir_hook; ++ + extern void copydir(char *fromdir, char *todir, bool recurse); + extern void copy_file(char *fromfile, char *tofile); + +diff --git a/src/include/storage/md.h b/src/include/storage/md.h +index 07fd1bb7d0..5294811bc8 100644 +--- a/src/include/storage/md.h ++++ b/src/include/storage/md.h +@@ -19,6 +19,13 @@ + #include "storage/smgr.h" + #include "storage/sync.h" + ++typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdextend_hook_type mdextend_hook; ++typedef void (*mdwrite_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook; ++ + /* md storage manager functionality */ + extern void mdinit(void); + extern void mdopen(SMgrRelation reln); +diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h +index e16ab8e711..88da9686eb 100644 +--- a/src/include/storage/sync.h ++++ b/src/include/storage/sync.h +@@ -50,6 +50,9 @@ typedef struct FileTag + uint32 segno; + } FileTag; + ++typedef void (*ProcessSyncRequests_hook_type) (void); ++extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; ++ + extern void InitSync(void); + extern void SyncPreCheckpoint(void); + extern void SyncPostCheckpoint(void); diff --git a/patches/master-ptrack-core.diff b/patches/master-ptrack-core.diff new file mode 100644 index 0000000..3491700 --- /dev/null +++ b/patches/master-ptrack-core.diff @@ -0,0 +1,309 @@ +commit a14ac459d71528c64df00c693e9c71ac70d3ba29 +Author: anastasia +Date: Mon Oct 19 14:53:06 2020 +0300 + + add ptrack 2.0 + +diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c +index 50ae1f16d0..721b926ad2 100644 +--- a/src/backend/replication/basebackup.c ++++ b/src/backend/replication/basebackup.c +@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ /* ++ * Skip all transient ptrack files, but do copy ptrack.map, since it may ++ * be successfully used immediately after backup. TODO: check, test? ++ */ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c +index 0cf598dd0c..c9c44a4ae7 100644 +--- a/src/backend/storage/file/copydir.c ++++ b/src/backend/storage/file/copydir.c +@@ -27,6 +27,8 @@ + #include "storage/copydir.h" + #include "storage/fd.h" + ++copydir_hook_type copydir_hook = NULL; ++ + /* + * copydir: copy a directory + * +@@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) + } + FreeDir(xldir); + ++ if (copydir_hook) ++ copydir_hook(todir); ++ + /* + * Be paranoid here and fsync all files to ensure the copy is really done. + * But if fsync is disabled, we're done. +diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c +index 0eacd461cd..c2ef404a1a 100644 +--- a/src/backend/storage/smgr/md.c ++++ b/src/backend/storage/smgr/md.c +@@ -87,6 +87,8 @@ typedef struct _MdfdVec + + static MemoryContext MdCxt; /* context for all MdfdVec objects */ + ++mdextend_hook_type mdextend_hook = NULL; ++mdwrite_hook_type mdwrite_hook = NULL; + + /* Populate a file tag describing an md.c segment file. */ + #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ +@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + register_dirty_segment(reln, forknum, v); + + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); ++ ++ if (mdextend_hook) ++ mdextend_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + + if (!skipFsync && !SmgrIsTemp(reln)) + register_dirty_segment(reln, forknum, v); ++ ++ if (mdwrite_hook) ++ mdwrite_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c +index 3ded2cdd71..3a596a59f7 100644 +--- a/src/backend/storage/sync/sync.c ++++ b/src/backend/storage/sync/sync.c +@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ + static CycleCtr sync_cycle_ctr = 0; + static CycleCtr checkpoint_cycle_ctr = 0; + ++ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; ++ + /* Intervals for calling AbsorbSyncRequests */ + #define FSYNCS_PER_ABSORB 10 + #define UNLINKS_PER_ABSORB 10 +@@ -420,6 +422,9 @@ ProcessSyncRequests(void) + CheckpointStats.ckpt_longest_sync = longest; + CheckpointStats.ckpt_agg_sync_time = total_elapsed; + ++ if (ProcessSyncRequests_hook) ++ ProcessSyncRequests_hook(); ++ + /* Flag successful completion of ProcessSyncRequests */ + sync_in_progress = false; + } +diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c +index 1683629ee3..d2fc154576 100644 +--- a/src/backend/utils/misc/guc.c ++++ b/src/backend/utils/misc/guc.c +@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; + static char *recovery_target_name_string; + static char *recovery_target_lsn_string; + +- + /* should be static, but commands/variable.c needs to get at this */ + char *role_string; + +diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c +index ffdc23945c..7ae95866ce 100644 +--- a/src/bin/pg_checksums/pg_checksums.c ++++ b/src/bin/pg_checksums/pg_checksums.c +@@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c +index 233441837f..cf7bd073bf 100644 +--- a/src/bin/pg_resetwal/pg_resetwal.c ++++ b/src/bin/pg_resetwal/pg_resetwal.c +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); + static void FindEndOfXLOG(void); + static void KillExistingXLOG(void); + static void KillExistingArchiveStatus(void); ++static void KillExistingPtrack(void); + static void WriteEmptyXLOG(void); + static void usage(void); + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) + RewriteControlFile(); + KillExistingXLOG(); + KillExistingArchiveStatus(); ++ KillExistingPtrack(); + WriteEmptyXLOG(); + + printf(_("Write-ahead log reset\n")); +@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) + } + } + ++/* ++ * Remove existing ptrack files ++ */ ++static void ++KillExistingPtrack(void) ++{ ++#define PTRACKDIR "global" ++ ++ DIR *xldir; ++ struct dirent *xlde; ++ char path[MAXPGPATH + sizeof(PTRACKDIR)]; ++ ++ xldir = opendir(PTRACKDIR); ++ if (xldir == NULL) ++ { ++ pg_log_error("could not open directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++ ++ while (errno = 0, (xlde = readdir(xldir)) != NULL) ++ { ++ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 || ++ strcmp(xlde->d_name, "ptrack.map") == 0 || ++ strcmp(xlde->d_name, "ptrack.map.tmp") == 0) ++ { ++ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name); ++ if (unlink(path) < 0) ++ { ++ pg_log_error("could not delete file \"%s\": %m", path); ++ exit(1); ++ } ++ } ++ } ++ ++ if (errno) ++ { ++ pg_log_error("could not read directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++ ++ if (closedir(xldir)) ++ { ++ pg_log_error("could not close directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++} ++ + + /* + * Write an empty XLOG file, containing only the checkpoint record +diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c +index fbb97b5cf1..6cd7f2ae3e 100644 +--- a/src/bin/pg_rewind/filemap.c ++++ b/src/bin/pg_rewind/filemap.c +@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h +index 72e3352398..5c2e016501 100644 +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -388,7 +388,7 @@ typedef enum ProcessingMode + NormalProcessing /* normal processing */ + } ProcessingMode; + +-extern ProcessingMode Mode; ++extern PGDLLIMPORT ProcessingMode Mode; + + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) + #define IsInitProcessingMode() (Mode == InitProcessing) +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 3c6f906683..a7355f7ad1 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif +diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h +index 5d28f59c1d..0d3f04d8af 100644 +--- a/src/include/storage/copydir.h ++++ b/src/include/storage/copydir.h +@@ -13,6 +13,9 @@ + #ifndef COPYDIR_H + #define COPYDIR_H + ++typedef void (*copydir_hook_type) (const char *path); ++extern PGDLLIMPORT copydir_hook_type copydir_hook; ++ + extern void copydir(char *fromdir, char *todir, bool recurse); + extern void copy_file(char *fromfile, char *tofile); + +diff --git a/src/include/storage/md.h b/src/include/storage/md.h +index 07fd1bb7d0..5294811bc8 100644 +--- a/src/include/storage/md.h ++++ b/src/include/storage/md.h +@@ -19,6 +19,13 @@ + #include "storage/smgr.h" + #include "storage/sync.h" + ++typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdextend_hook_type mdextend_hook; ++typedef void (*mdwrite_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook; ++ + /* md storage manager functionality */ + extern void mdinit(void); + extern void mdopen(SMgrRelation reln); +diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h +index e16ab8e711..88da9686eb 100644 +--- a/src/include/storage/sync.h ++++ b/src/include/storage/sync.h +@@ -50,6 +50,9 @@ typedef struct FileTag + uint32 segno; + } FileTag; + ++typedef void (*ProcessSyncRequests_hook_type) (void); ++extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; ++ + extern void InitSync(void); + extern void SyncPreCheckpoint(void); + extern void SyncPostCheckpoint(void); From 24fe60ad2db70cbf9f212875e3966a6ed7d2b20e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Wed, 13 Oct 2021 23:14:31 +0300 Subject: [PATCH 2/2] running tests using python 3 (instead of python 2) --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 90654cc..b2ab300 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -105,7 +105,7 @@ else # Setup python environment echo "############### Setting up python env" - virtualenv pyenv + virtualenv --python=/usr/bin/python3 pyenv source pyenv/bin/activate pip install testgres==1.8.2 @@ -118,10 +118,10 @@ else if [ "$TEST_CASE" = "all" ]; then # Run all pg_probackup ptrack tests - python -m unittest -v tests.ptrack || status=$? + python3 -m unittest -v tests.ptrack || status=$? else for i in `seq $TEST_REPEATS`; do - python -m unittest -v tests.ptrack.PtrackTest.$TEST_CASE || status=$? + python3 -m unittest -v tests.ptrack.PtrackTest.$TEST_CASE || status=$? done fi