diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f05773a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,94 @@ +name: Test + +on: + push: + branches: + - "**" + pull_request: + branches: + - main + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + Test: + strategy: + matrix: + # pg_version: [15] + pg_version: [11, 12, 13, 14, 15] + os: [ubuntu-22.04] + # tests: [tap] + tests: [tap, python] + # test_mode: [normal, legacy, paranoia] + test_mode: [normal, paranoia] + exclude: + - tests: tap + test_mode: paranoia + - tests: python + test_mode: normal + - tests: python + test_mode: legacy + fail-fast: false + name: ${{ format('Ptrack ({0}, PostgreSQL {1}, {2} tests, {3} mode)', matrix.os, matrix.pg_version, matrix.tests, matrix.test_mode) }} + container: + image: ${{ format('ghcr.io/postgres-dev/{0}:1.0', matrix.os) }} + env: + PG_BRANCH: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + PGDATA: $HOME/data + TEST_MODE: ${{ matrix.test_mode }} + options: --privileged + steps: + - name: Get Postgres sources + uses: actions/checkout@v3 + with: + repository: postgres/postgres + ref: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + path: postgres + - name: Get Ptrack sources + uses: actions/checkout@v3 + with: + path: ptrack + - name: Get Pg_probackup sources + uses: actions/checkout@v3 + with: + repository: postgrespro/pg_probackup + path: pg_probackup + - name: Apply ptrack patches + run: make patch top_builddir=../postgres + working-directory: ptrack + - name: Install Postgres + run: | + make install-postgres top_builddir=$GITHUB_WORKSPACE/postgres prefix=$HOME/pgsql && + echo $HOME/pgsql/bin >> $GITHUB_PATH + working-directory: ptrack + - name: Install Ptrack + run: make install USE_PGXS=1 PG_CPPFLAGS=-coverage SHLIB_LINK=-coverage + working-directory: ptrack + - name: Install Pg_probackup + run: make install-pg-probackup USE_PGXS=1 top_srcdir=../postgres + working-directory: ptrack + shell: bash {0} + - name: Install additional packages + run: | + apt update && + apt install -y python3-pip python3-six python3-pytest python3-pytest-xdist curl && + pip3 install --no-input testgres + # All steps have been so far executed by root but ptrack tests run from an + # unprivileged user so change some permissions + - name: Adjust the permissions of ptrack test folders + run: | + mkdir pg_probackup/tests/tmp_dirs + chown -R "dev:" pg_probackup ptrack + - name: Test + run: make test-${{ matrix.tests }} USE_PGXS=1 + working-directory: ptrack + shell: runuser dev {0} + - name: Collect coverage results + run: make coverage + working-directory: ptrack + shell: runuser dev {0} + - name: Upload coverage results to Codecov + uses: codecov/codecov-action@v3 + with: + working-directory: ptrack + runs-on: ubuntu-latest diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b3698e1..0000000 --- a/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ -os: linux - -dist: bionic - -language: c - -services: - - docker - -install: - - ./make_dockerfile.sh - - docker-compose build - -script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - -notifications: - email: - on_success: change - on_failure: always - -# keep in sync with codecov.yml number of builds -env: - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=all - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=tap - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=all - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..ed4d0eb --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,22 @@ +# Authors + +This list is sorted by the number of commits per contributor in _descending_ order. + +Avatar|Contributor|Contributions +:-:|---|:-: +@ololobus|[@ololobus](https://github.com/ololobus)|62 +@funny-falcon|[@funny-falcon](https://github.com/funny-falcon)|15 +@alubennikova|[@alubennikova](https://github.com/alubennikova)|9 +@kulaginm|[@kulaginm](https://github.com/kulaginm)|5 +@daniel-95|[@daniel-95](https://github.com/daniel-95)|4 +@ziva777|[@ziva777](https://github.com/ziva777)|2 +@vegebird|[@vegebird](https://github.com/vegebird)|2 +@kovdb75|[@kovdb75](https://github.com/kovdb75)|1 +@MarinaPolyakova|[@MarinaPolyakova](https://github.com/MarinaPolyakova)|1 +@rzharkov|[@rzharkov](https://github.com/rzharkov)|1 +@vbwagner|[@vbwagner](https://github.com/vbwagner)|1 +@waaeer|[@waaeer](https://github.com/waaeer)|1 + +--- + +Auto-generated by [gaocegege/maintainer](https://github.com/maintainer-org/maintainer) on 2023-08-03. diff --git a/Dockerfile.in b/Dockerfile.in deleted file mode 100644 index 39541da..0000000 --- a/Dockerfile.in +++ /dev/null @@ -1,25 +0,0 @@ -FROM ololobus/postgres-dev:stretch - -USER root -RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv - -# Environment -ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} -ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin -ENV MODE=${MODE} TEST_CASE=${TEST_CASE} TEST_REPEATS=${TEST_REPEATS} - -# Make directories -RUN mkdir -p /pg/testdir - -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -COPY . /pg/testdir -WORKDIR /pg/testdir - -# Grant privileges -RUN chown -R postgres:postgres /pg/testdir - -USER postgres -ENTRYPOINT /run.sh diff --git a/LICENSE b/LICENSE index 057f651..c1393f3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ PostgreSQL License -Copyright (c) 2019-2020, Postgres Professional +Copyright (c) 2019-2023, Postgres Professional Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is diff --git a/Makefile b/Makefile index ba9ce1d..499067a 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ + # contrib/ptrack/Makefile MODULE_big = ptrack @@ -5,18 +6,80 @@ OBJS = ptrack.o datapagemap.o engine.o $(WIN32RES) PGFILEDESC = "ptrack - block-level incremental backup engine" EXTENSION = ptrack -EXTVERSION = 2.2 -DATA = ptrack--2.1.sql ptrack--2.0--2.1.sql ptrack--2.1--2.2.sql +EXTVERSION = 2.4 +DATA = ptrack--2.1.sql ptrack--2.0--2.1.sql ptrack--2.1--2.2.sql ptrack--2.2--2.3.sql \ + ptrack--2.3--2.4.sql TAP_TESTS = 1 -ifdef USE_PGXS +# This line to link with pgport.lib on Windows compilation +# with Mkvcbuild.pm on PGv15+ +PG_LIBS_INTERNAL += $(libpq_pgport) + PG_CONFIG ?= pg_config + +ifdef USE_PGXS PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else -subdir = contrib/ptrack top_builddir = ../.. +# Makefile.global is a build artifact and initially may not be available +ifneq ($(wildcard $(top_builddir)/src/Makefile.global), ) include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif +endif + +# Assuming make is started in the ptrack directory +patch: + @cd $(top_builddir) && \ + echo Applying the ptrack patch... && \ + git apply --3way -v $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff +ifeq ($(MODE), paranoia) + @echo Applying turn-off-hint-bits.diff for the paranoia mode... && \ + git apply --3way -v $(CURDIR)/patches/turn-off-hint-bits.diff +endif + +NPROC ?= $(shell nproc) +prefix := $(abspath $(top_builddir)/pgsql) +TEST_MODE ?= normal +# Postgres Makefile skips some targets depending on the MAKELEVEL variable so +# reset it when calling install targets as if they are started directly from the +# command line +install-postgres: + @cd $(top_builddir) && \ + if [ "$(TEST_MODE)" = legacy ]; then \ + ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet; \ + else \ + ./configure --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests; \ + fi && \ + $(MAKE) -sj $(NPROC) install MAKELEVEL=0 && \ + $(MAKE) -sj $(NPROC) -C contrib/ install MAKELEVEL=0 + +# Now when Postgres is built call all remainig targets with USE_PGXS=1 + +test-tap: +ifeq ($(TEST_MODE), legacy) + setarch x86_64 --addr-no-randomize $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +else + $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +endif + +pg_probackup_dir = ../pg_probackup +# Pg_probackup's Makefile uses top_srcdir when building via PGXS so set it when calling this target +# At the moment building pg_probackup with multiple threads may run some jobs too early and end with an error so do not set the -j option +install-pg-probackup: + $(MAKE) -C $(pg_probackup_dir) install USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) top_srcdir=$(top_srcdir) + +test-python: + cd $(pg_probackup_dir); \ + env="PG_PROBACKUP_PTRACK=ON PG_CONFIG=$(PG_CONFIG)"; \ + if [ "$(TEST_MODE)" = normal ]; then \ + env="$$env PG_PROBACKUP_TEST_BASIC=ON"; \ + elif [ "$(TEST_MODE)" = paranoia ]; then \ + env="$$env PG_PROBACKUP_PARANOIA=ON"; \ + fi; \ + env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(NPROC))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + +coverage: + gcov *.c *.h diff --git a/README.md b/README.md index 0d6d232..1dd4a94 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Build Status](https://travis-ci.com/postgrespro/ptrack.svg?branch=master)](https://travis-ci.com/postgrespro/ptrack) -[![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) +[![Test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg)](https://github.com/postgrespro/ptrack/actions/workflows/test.yml) +[![Codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) # ptrack @@ -12,44 +12,66 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files. -This extension is compatible with PostgreSQL [11](https://github.com/postgrespro/ptrack/blob/master/patches/REL_11_STABLE-ptrack-core.diff), [12](https://github.com/postgrespro/ptrack/blob/master/patches/REL_12_STABLE-ptrack-core.diff), [13](https://github.com/postgrespro/ptrack/blob/master/patches/REL_13_STABLE-ptrack-core.diff). +This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff). ## Installation -1) Get latest `ptrack` sources: +1) Specify the PostgreSQL branch to work with: ```shell -git clone https://github.com/postgrespro/ptrack.git +export PG_BRANCH=REL_15_STABLE ``` -2) Get latest PostgreSQL sources: +2) Get the latest PostgreSQL sources: ```shell -git clone https://github.com/postgres/postgres.git -b REL_12_STABLE && cd postgres +git clone https://github.com/postgres/postgres.git -b $PG_BRANCH ``` -3) Apply PostgreSQL core patch: +3) Get the latest `ptrack` sources: ```shell -git apply -3 ../ptrack/patches/REL_12_STABLE-ptrack-core.diff +git clone https://github.com/postgrespro/ptrack.git postgres/contrib/ptrack ``` -4) Compile and install PostgreSQL +4) Change to the `ptrack` directory: -5) Set `ptrack.map_size` (in MB) +```shell +cd postgres/contrib/ptrack +``` + +5) Apply the PostgreSQL core patch: + +```shell +make patch +``` + +6) Compile and install PostgreSQL: + +```shell +make install-postgres prefix=$PWD/pgsql # or some other prefix of your choice +``` + +7) Add the newly created binaries to the PATH: ```shell -echo "shared_preload_libraries = 'ptrack'" >> postgres_data/postgresql.conf -echo "ptrack.map_size = 64" >> postgres_data/postgresql.conf +export PATH=$PWD/pgsql/bin:$PATH ``` -6) Compile and install `ptrack` extension +8) Compile and install `ptrack`: ```shell -USE_PGXS=1 make -C /path/to/ptrack/ install +make install USE_PGXS=1 ``` -7) Run PostgreSQL and create `ptrack` extension +9) Set `ptrack.map_size` (in MB): + +```shell +echo "shared_preload_libraries = 'ptrack'" >> /postgresql.conf +echo "ptrack.map_size = 64" >> /postgresql.conf +``` + +10) Run PostgreSQL and create the `ptrack` extension: ```sql postgres=# CREATE EXTENSION ptrack; @@ -57,7 +79,7 @@ postgres=# CREATE EXTENSION ptrack; ## Configuration -The only one configurable option is `ptrack.map_size` (in MB). Default is `-1`, which means `ptrack` is turned off. In order to reduce number of false positives it is recommended to set `ptrack.map_size` to `1 / 1000` of expected `PGDATA` size (i.e. `1000` for a 1 TB database). +The only one configurable option is `ptrack.map_size` (in MB). Default is `0`, which means `ptrack` is turned off. In order to reduce number of false positives it is recommended to set `ptrack.map_size` to `1 / 1000` of expected `PGDATA` size (i.e. `1000` for a 1 TB database). To disable `ptrack` and clean up all remaining service files set `ptrack.map_size` to `0`. @@ -74,7 +96,7 @@ Usage example: postgres=# SELECT ptrack_version(); ptrack_version ---------------- - 2.2 + 2.4 (1 row) postgres=# SELECT ptrack_init_lsn(); @@ -102,28 +124,43 @@ postgres=# SELECT * FROM ptrack_get_change_stat('0/285C8C8'); ## Upgrading -Usually, you have to only install new version of `ptrack` and do `ALTER EXTENSION 'ptrack' UPDATE;`. However, some specific actions may be required as well: +Usually, you have to only install new version of `ptrack` and do `ALTER EXTENSION ptrack UPDATE;`. However, some specific actions may be required as well: #### Upgrading from 2.0.0 to 2.1.*: * Put `shared_preload_libraries = 'ptrack'` into `postgresql.conf`. * Rename `ptrack_map_size` to `ptrack.map_size`. -* Do `ALTER EXTENSION 'ptrack' UPDATE;`. +* Do `ALTER EXTENSION ptrack UPDATE;`. * Restart your server. #### Upgrading from 2.1.* to 2.2.*: Since version 2.2 we use a different algorithm for tracking changed pages. Thus, data recorded in the `ptrack.map` using pre 2.2 versions of `ptrack` is incompatible with newer versions. After extension upgrade and server restart old `ptrack.map` will be discarded with `WARNING` and initialized from the scratch. +#### Upgrading from 2.2.* to 2.3.*: + +* Stop your server +* Update ptrack binaries +* Remove global/ptrack.map.mmap if it exist in server data directory +* Start server +* Do `ALTER EXTENSION ptrack UPDATE;`. + +#### Upgrading from 2.3.* to 2.4.*: + +* Stop your server +* Update ptrack binaries +* Start server +* Do `ALTER EXTENSION ptrack UPDATE;`. + ## Limitations 1. You can only use `ptrack` safely with `wal_level >= 'replica'`. Otherwise, you can lose tracking of some changes if crash-recovery occurs, since [certain commands are designed not to write WAL at all if wal_level is minimal](https://www.postgresql.org/docs/12/populate.html#POPULATE-PITR), but we only durably flush `ptrack` map at checkpoint time. 2. The only one production-ready backup utility, that fully supports `ptrack` is [pg_probackup](https://github.com/postgrespro/pg_probackup). -3. Currently, you cannot resize `ptrack` map in runtime, only on postmaster start. Also, you will loose all tracked changes, so it is recommended to do so in the maintainance window and accompany this operation with full backup. See [TODO](#TODO) for details. +3. You cannot resize `ptrack` map in runtime, only on postmaster start. Also, you will loose all tracked changes, so it is recommended to do so in the maintainance window and accompany this operation with full backup. -4. You will need up to `ptrack.map_size * 3` of additional disk space, since `ptrack` uses two additional temporary files for durability purpose. See [Architecture section](#Architecture) for details. +4. You will need up to `ptrack.map_size * 2` of additional disk space, since `ptrack` uses additional temporary file for durability purpose. See [Architecture section](#Architecture) for details. ## Benchmarks @@ -131,11 +168,10 @@ Briefly, an overhead of using `ptrack` on TPS usually does not exceed a couple o ## Architecture -We use a single shared hash table in `ptrack`, which is mapped in memory from the file on disk using `mmap`. Due to the fixed size of the map there may be false positives (when some block is marked as changed without being actually modified), but not false negative results. However, these false postives may be completely eliminated by setting a high enough `ptrack.map_size`. +We use a single shared hash table in `ptrack`. Due to the fixed size of the map there may be false positives (when some block is marked as changed without being actually modified), but not false negative results. However, these false postives may be completely eliminated by setting a high enough `ptrack.map_size`. -All reads/writes are made using atomic operations on `uint64` entries, so the map is completely lockless during the normal PostgreSQL operation. Because we do not use locks for read/write access and cannot control `mmap` eviction back to disk, `ptrack` keeps a map (`ptrack.map`) since the last checkpoint intact and uses up to 2 additional temporary files: +All reads/writes are made using atomic operations on `uint64` entries, so the map is completely lockless during the normal PostgreSQL operation. Because we do not use locks for read/write access, `ptrack` keeps a map (`ptrack.map`) since the last checkpoint intact and uses up to 1 additional temporary file: -* working copy `ptrack.map.mmap` for doing `mmap` on it (there is a [TODO](#TODO) item); * temporary file `ptrack.map.tmp` to durably replace `ptrack.map` during checkpoint. Map is written on disk at the end of checkpoint atomically block by block involving the CRC32 checksum calculation that is checked on the next whole map re-read after crash-recovery or restart. @@ -144,30 +180,67 @@ To gather the whole changeset of modified blocks in `ptrack_get_pagemapset()` we ## Contribution -Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare), [fill up issues](https://github.com/postgrespro/ptrack/issues/new), or just reach one of us directly (e.g. <[Alexey Kondratov](mailto:a.kondratov@postgrespro.ru?subject=[GitHub]%20Ptrack), [@ololobus](https://github.com/ololobus)>) if you are interested in `ptrack`. +Feel free to [send a pull request](https://github.com/postgrespro/ptrack/compare), [create an issue](https://github.com/postgrespro/ptrack/issues/new) or [reach us by e-mail](mailto:team-wd40@lists.postgrespro.ru??subject=[GitHub]%20Ptrack) if you are interested in `ptrack`. + +## Tests + +All changes of the source code in this repository are checked by CI - see commit statuses and the project status badge. You can also run tests locally by executing a few Makefile targets. -### Tests +### Prerequisites -Everything is tested automatically with [travis-ci.com](https://travis-ci.com/postgrespro/ptrack) and [codecov.io](https://codecov.io/gh/postgrespro/ptrack), but you can also run tests locally via `Docker`: +To run Python tests install the following packages: -```sh -export PG_VERSION=12 -export PG_BRANCH=REL_12_STABLE -export TEST_CASE=all -export MODE=paranoia +OS packages: + - python3-pip + - python3-six + - python3-pytest + - python3-pytest-xdist -./make_dockerfile.sh +PIP packages: + - testgres -docker-compose build -docker-compose run tests +For example, for Ubuntu: + +```shell +sudo apt update +sudo apt install python3-pip python3-six python3-pytest python3-pytest-xdist +sudo pip3 install testgres +``` + +### Testing + +Install PostgreSQL and ptrack as described in [Installation](#installation), install the testing prerequisites, then do (assuming the current directory is `ptrack`): +```shell +git clone https://github.com/postgrespro/pg_probackup.git ../pg_probackup # clone the repository into postgres/contrib/pg_probackup +# remember to export PATH=/path/to/pgsql/bin:$PATH +make install-pg-probackup USE_PGXS=1 top_srcdir=../.. +make test-tap USE_PGXS=1 +make test-python ``` -Available test modes (`MODE`) are `basic` (default) and `paranoia` (per-block checksum comparison of `PGDATA` content before and after backup-restore process). Available test cases (`TEST_CASE`) are `tap` (minimalistic PostgreSQL [tap test](https://github.com/postgrespro/ptrack/blob/master/t/001_basic.pl)), `all` or any specific [pg_probackup test](https://github.com/postgrespro/pg_probackup/blob/master/tests/ptrack.py), e.g. `test_ptrack_simple`. +If `pg_probackup` is not located in `postgres/contrib` then additionally specify the path to the `pg_probackup` directory when building `pg_probackup`: +```shell +make install-pg-probackup USE_PGXS=1 top_srcdir=/path/to/postgres pg_probackup_dir=/path/to/pg_probackup +``` + +You can use a public Docker image which already has the necessary build environment (but not the testing prerequisites): + +```shell +docker run -e USER_ID=`id -u` -it -v $PWD:/work --name=ptrack ghcr.io/postgres-dev/ubuntu-22.04:1.0 +dev@a033797d2f73:~$ +``` + +## Environment variables + +| Variable | Possible values | Required | Default value | Description | +| - | - | - | - | - | +| NPROC | An integer greater than 0 | No | Output of `nproc` | The number of threads used for building and running tests | +| PG_CONFIG | File path | No | pg_config (from the PATH) | The path to the `pg_config` binary | +| TESTS | A Pytest filter expression | No | Not set (run all Python tests) | A filter to include only selected tests into the run. See the Pytest `-k` option for more information. This variable is only applicable to `test-python` for the tests located in [tests](https://github.com/postgrespro/pg_probackup/tree/master/tests). | +| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backup and after the restoration. This mode is only applicable to `test-python`.| ### TODO -* Use POSIX `shm_open()` instead of `open()` to do not create an additional working copy of `ptrack` map file. * Should we introduce `ptrack.map_path` to allow `ptrack` service files storage outside of `PGDATA`? Doing that we will avoid patching PostgreSQL binary utilities to ignore `ptrack.map.*` files. * Can we resize `ptrack` map on restart but keep the previously tracked changes? -* Can we resize `ptrack` map dynamicaly? * Can we write a formal proof, that we never loose any modified page with `ptrack`? With TLA+? diff --git a/codecov.yml b/codecov.yml index fe3b308..00b744e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,9 @@ codecov: notify: - after_n_builds: 12 # keep in sync with .travis.yml number of builds + # must be equal to the total number of parallel jobs in a CI pipeline + # (Postgres versions x test types x test modes x OSes minus excluded + # combinations) + after_n_builds: 10 # datapagemap.c/.h are copied from Postgres, so let's remove it # from report. Otherwise, we would have to remove some currently diff --git a/datapagemap.h b/datapagemap.h index 455705f..9b730da 100644 --- a/datapagemap.h +++ b/datapagemap.h @@ -9,7 +9,6 @@ #ifndef DATAPAGEMAP_H #define DATAPAGEMAP_H -#include "storage/relfilenode.h" #include "storage/block.h" struct datapagemap diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 544e59f..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,3 +0,0 @@ -tests: - privileged: true - build: . diff --git a/engine.c b/engine.c index bef0b2b..dfd7c84 100644 --- a/engine.c +++ b/engine.c @@ -2,14 +2,14 @@ * engine.c * Block level incremental backup engine core * - * Copyright (c) 2019-2020, Postgres Professional + * Copyright (c) 2019-2022, Postgres Professional * * IDENTIFICATION * ptrack/engine.c * * INTERFACE ROUTINES (PostgreSQL side) * ptrackMapInit() --- allocate new shared ptrack_map - * ptrackMapAttach() --- attach to the existing ptrack_map + * ptrackCleanFiles() --- remove ptrack files * assign_ptrack_map_size() --- ptrack_map_size GUC assign callback * ptrack_walkdir() --- walk directory and mark all blocks of all * data files in ptrack_map @@ -29,6 +29,10 @@ #include "access/htup_details.h" #include "access/parallel.h" #include "access/xlog.h" +#if PG_VERSION_NUM >= 150000 +#include "access/xlogrecovery.h" +#include "storage/fd.h" +#endif #include "catalog/pg_tablespace.h" #include "miscadmin.h" #include "port/pg_crc32c.h" @@ -55,7 +59,7 @@ ptrack_file_exists(const char *path) { struct stat st; - AssertArg(path != NULL); + Assert(path != NULL); if (stat(path, &st) == 0) return S_ISDIR(st.st_mode) ? false : true; @@ -88,160 +92,110 @@ ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) } /* - * Delete ptrack file and free the memory when ptrack is disabled. + * Delete ptrack files when ptrack is disabled. * - * This is performed by postmaster at start or by checkpointer, + * This is performed by postmaster at start, * so that there are no concurrent delete issues. */ -static void -ptrackCleanFilesAndMap(void) +void +ptrackCleanFiles(void) { char ptrack_path[MAXPGPATH]; - char ptrack_mmap_path[MAXPGPATH]; char ptrack_path_tmp[MAXPGPATH]; sprintf(ptrack_path, "%s/%s", DataDir, PTRACK_PATH); - sprintf(ptrack_mmap_path, "%s/%s", DataDir, PTRACK_MMAP_PATH); sprintf(ptrack_path_tmp, "%s/%s", DataDir, PTRACK_PATH_TMP); - elog(DEBUG1, "ptrack: clean files and map"); + elog(DEBUG1, "ptrack: clean map files"); if (ptrack_file_exists(ptrack_path_tmp)) durable_unlink(ptrack_path_tmp, LOG); if (ptrack_file_exists(ptrack_path)) durable_unlink(ptrack_path, LOG); - - if (ptrack_map != NULL) - { -#ifdef WIN32 - if (!UnmapViewOfFile(ptrack_map)) -#else - if (!munmap(ptrack_map, sizeof(ptrack_map))) -#endif - elog(LOG, "could not unmap ptrack_map"); - - ptrack_map = NULL; - } - - if (ptrack_file_exists(ptrack_mmap_path)) - durable_unlink(ptrack_mmap_path, LOG); } /* - * Copy PTRACK_PATH file to special temporary file PTRACK_MMAP_PATH used for mapping, - * or create new file, if there was no PTRACK_PATH file on disk. - * - * Map the content of PTRACK_MMAP_PATH file into memory structure 'ptrack_map' using mmap. + * Read ptrack map file into shared memory pointed by ptrack_map. + * This function is called only at startup, + * so data is read directly (without synchronization). */ -void -ptrackMapInit(void) +static bool +ptrackMapReadFromFile(const char *ptrack_path) { - int ptrack_fd; - pg_crc32c crc; - pg_crc32c *file_crc; - char ptrack_path[MAXPGPATH]; - char ptrack_mmap_path[MAXPGPATH]; - struct stat stat_buf; - bool is_new_map = true; - - elog(DEBUG1, "ptrack init"); + elog(DEBUG1, "ptrack read map"); - /* We do it at server start, so the map must be not allocated yet. */ - Assert(ptrack_map == NULL); - - if (ptrack_map_size == 0) - return; + /* Do actual file read */ + { + int ptrack_fd; + size_t readed; - sprintf(ptrack_path, "%s/%s", DataDir, PTRACK_PATH); - sprintf(ptrack_mmap_path, "%s/%s", DataDir, PTRACK_MMAP_PATH); + ptrack_fd = BasicOpenFile(ptrack_path, O_RDWR | PG_BINARY); -ptrack_map_reinit: + if (ptrack_fd < 0) + elog(ERROR, "ptrack read map: failed to open map file \"%s\": %m", ptrack_path); - /* Remove old PTRACK_MMAP_PATH file, if exists */ - if (ptrack_file_exists(ptrack_mmap_path)) - durable_unlink(ptrack_mmap_path, LOG); + readed = 0; + do + { + ssize_t last_readed; - if (stat(ptrack_path, &stat_buf) == 0 && - stat_buf.st_size != PtrackActualSize) - { - elog(WARNING, "ptrack init: unexpected \"%s\" file size %zu != " UINT64_FORMAT ", deleting", - ptrack_path, (Size) stat_buf.st_size, PtrackActualSize); - durable_unlink(ptrack_path, LOG); + /* + * Try to read as much as possible + * (linux guaranteed only 0x7ffff000 bytes in one read + * operation, see read(2)) + */ + last_readed = read(ptrack_fd, (char *) ptrack_map + readed, PtrackActualSize - readed); + + if (last_readed > 0) + { + readed += last_readed; + } + else if (last_readed == 0) + { + /* + * We don't try to read more that PtrackActualSize and + * file size was already checked in ptrackMapInit() + */ + elog(ERROR, "ptrack read map: unexpected end of file while reading map file \"%s\", expected to read %zu, but read only %zu bytes", + ptrack_path, (size_t)PtrackActualSize, readed); + } + else if (last_readed < 0 && errno != EINTR) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("ptrack read map: could not read map file \"%s\": %m", ptrack_path))); + close(ptrack_fd); + return false; + } + } while (readed < PtrackActualSize); + + close(ptrack_fd); } - /* - * If on-disk PTRACK_PATH file is present and has expected size, copy it - * to read and restore state. - */ - if (stat(ptrack_path, &stat_buf) == 0) + /* Check PTRACK_MAGIC */ + if (strcmp(ptrack_map->magic, PTRACK_MAGIC) != 0) { - copy_file(ptrack_path, ptrack_mmap_path); - is_new_map = false; /* flag to check map file format and checksum */ - ptrack_fd = BasicOpenFile(ptrack_mmap_path, O_RDWR | PG_BINARY); + elog(WARNING, "ptrack read map: wrong map format of file \"%s\"", ptrack_path); + return false; } - else - /* Create new file for PTRACK_MMAP_PATH */ - ptrack_fd = BasicOpenFile(ptrack_mmap_path, O_RDWR | O_CREAT | PG_BINARY); - - if (ptrack_fd < 0) - elog(ERROR, "ptrack init: failed to open map file \"%s\": %m", ptrack_mmap_path); -#ifdef WIN32 + /* Check ptrack version inside old ptrack map */ + if (ptrack_map->version_num != PTRACK_MAP_FILE_VERSION_NUM) { - HANDLE mh = CreateFileMapping((HANDLE) _get_osfhandle(ptrack_fd), - NULL, - PAGE_READWRITE, - 0, - (DWORD) PtrackActualSize, - NULL); - - if (mh == NULL) - elog(ERROR, "ptrack init: failed to create file mapping: %m"); - - ptrack_map = (PtrackMap) MapViewOfFile(mh, FILE_MAP_ALL_ACCESS, 0, 0, 0); - if (ptrack_map == NULL) - { - CloseHandle(mh); - elog(ERROR, "ptrack init: failed to mmap ptrack file: %m"); - } + ereport(WARNING, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("ptrack read map: map format version %d in the file \"%s\" is incompatible with file format of extension %d", + ptrack_map->version_num, ptrack_path, PTRACK_MAP_FILE_VERSION_NUM), + errdetail("Deleting file \"%s\" and reinitializing ptrack map.", ptrack_path))); + return false; } -#else - if (ftruncate(ptrack_fd, PtrackActualSize) < 0) - elog(ERROR, "ptrack init: failed to truncate file: %m"); - - ptrack_map = (PtrackMap) mmap(NULL, PtrackActualSize, - PROT_READ | PROT_WRITE, MAP_SHARED, - ptrack_fd, 0); - if (ptrack_map == MAP_FAILED) - elog(ERROR, "ptrack init: failed to mmap file: %m"); -#endif - if (!is_new_map) + /* Check CRC */ { - XLogRecPtr init_lsn; - - /* Check PTRACK_MAGIC */ - if (strcmp(ptrack_map->magic, PTRACK_MAGIC) != 0) - elog(ERROR, "ptrack init: wrong map format of file \"%s\"", ptrack_path); - - /* Check ptrack version inside old ptrack map */ - if (ptrack_map->version_num != PTRACK_VERSION_NUM) - { - ereport(WARNING, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("ptrack init: map format version %d in the file \"%s\" is incompatible with loaded version %d", - ptrack_map->version_num, ptrack_path, PTRACK_VERSION_NUM), - errdetail("Deleting file \"%s\" and reinitializing ptrack map.", ptrack_path))); - - /* Clean up everything and try again */ - ptrackCleanFilesAndMap(); + pg_crc32c crc; + pg_crc32c *file_crc; - is_new_map = true; - goto ptrack_map_reinit; - } - - /* Check CRC */ INIT_CRC32C(crc); COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); FIN_CRC32C(crc); @@ -252,88 +206,85 @@ ptrackMapInit(void) * Read ptrack map values without atomics during initialization, since * postmaster is the only user right now. */ - init_lsn = ptrack_map->init_lsn.value; - elog(DEBUG1, "ptrack init: crc %u, file_crc %u, init_lsn %X/%X", - crc, *file_crc, (uint32) (init_lsn >> 32), (uint32) init_lsn); + elog(DEBUG1, "ptrack read map: crc %u, file_crc %u, init_lsn %X/%X", + crc, *file_crc, (uint32) (ptrack_map->init_lsn.value >> 32), (uint32) ptrack_map->init_lsn.value); - /* TODO: Handle this error. Probably we can just recreate the file */ if (!EQ_CRC32C(*file_crc, crc)) { - ereport(ERROR, + ereport(WARNING, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("ptrack init: incorrect checksum of file \"%s\"", ptrack_path), - errhint("Delete \"%s\" and start the server again.", ptrack_path))); + errmsg("ptrack read map: incorrect checksum of file \"%s\"", ptrack_path), + errdetail("Deleting file \"%s\" and reinitializing ptrack map.", ptrack_path))); + return false; } } - else - { - memcpy(ptrack_map->magic, PTRACK_MAGIC, PTRACK_MAGIC_SIZE); - ptrack_map->version_num = PTRACK_VERSION_NUM; - } + return true; } /* - * Map must be already initialized by postmaster at start. - * mmap working copy of ptrack_map. + * Read PTRACK_PATH file into already allocated shared memory, check header and checksum + * or create new file, if there was no PTRACK_PATH file on disk. */ void -ptrackMapAttach(void) +ptrackMapInit(void) { - char ptrack_mmap_path[MAXPGPATH]; - int ptrack_fd; + char ptrack_path[MAXPGPATH]; struct stat stat_buf; + bool is_new_map = true; - elog(DEBUG1, "ptrack attach"); - - /* We do it at process start, so the map must be not allocated yet. */ - Assert(ptrack_map == NULL); + elog(DEBUG1, "ptrack init"); if (ptrack_map_size == 0) return; - sprintf(ptrack_mmap_path, "%s/%s", DataDir, PTRACK_MMAP_PATH); - if (!ptrack_file_exists(ptrack_mmap_path)) - { - elog(WARNING, "ptrack attach: '%s' file doesn't exist ", ptrack_mmap_path); - return; - } - - if (stat(ptrack_mmap_path, &stat_buf) == 0 && - stat_buf.st_size != PtrackActualSize) - elog(ERROR, "ptrack attach: ptrack_map_size doesn't match size of the file \"%s\"", ptrack_mmap_path); - - ptrack_fd = BasicOpenFile(ptrack_mmap_path, O_RDWR | PG_BINARY); - if (ptrack_fd < 0) - elog(ERROR, "ptrack attach: failed to open ptrack map file \"%s\": %m", ptrack_mmap_path); + sprintf(ptrack_path, "%s/%s", DataDir, PTRACK_PATH); - elog(DEBUG1, "ptrack attach: before mmap"); -#ifdef WIN32 + if (stat(ptrack_path, &stat_buf) == 0) { - HANDLE mh = CreateFileMapping((HANDLE) _get_osfhandle(ptrack_fd), - NULL, - PAGE_READWRITE, - 0, - (DWORD) PtrackActualSize, - NULL); - - if (mh == NULL) - elog(ERROR, "ptrack attach: failed to create file mapping: %m"); - - ptrack_map = (PtrackMap) MapViewOfFile(mh, FILE_MAP_ALL_ACCESS, 0, 0, 0); - if (ptrack_map == NULL) + elog(DEBUG3, "ptrack init: map \"%s\" detected, trying to load", ptrack_path); + if (stat_buf.st_size != PtrackActualSize) + { + elog(WARNING, "ptrack init: unexpected \"%s\" file size %zu != " UINT64_FORMAT ", deleting", + ptrack_path, (Size) stat_buf.st_size, PtrackActualSize); + durable_unlink(ptrack_path, LOG); + } + else if (ptrackMapReadFromFile(ptrack_path)) { - CloseHandle(mh); - elog(ERROR, "ptrack attach: failed to mmap ptrack file: %m"); + is_new_map = false; + } + else + { + /* + * ptrackMapReadFromFile failed + * this can be crc mismatch, version mismatch and other errors + * We treat it as non fatal and create new map in memory, + * that will be written on disk on checkpoint + */ + elog(WARNING, "ptrack init: broken map file \"%s\", deleting", + ptrack_path); + durable_unlink(ptrack_path, LOG); } } -#else - ptrack_map = (PtrackMap) mmap(NULL, PtrackActualSize, - PROT_READ | PROT_WRITE, MAP_SHARED, - ptrack_fd, 0); - if (ptrack_map == MAP_FAILED) - elog(ERROR, "ptrack attach: failed to mmap ptrack file: %m"); -#endif + + /* + * Initialyze memory for new map + */ + if (is_new_map) + { + memcpy(ptrack_map->magic, PTRACK_MAGIC, PTRACK_MAGIC_SIZE); + ptrack_map->version_num = PTRACK_MAP_FILE_VERSION_NUM; + ptrack_map->init_lsn.value = InvalidXLogRecPtr; + /* + * Fill entries with InvalidXLogRecPtr + * (InvalidXLogRecPtr is actually 0) + */ + memset(ptrack_map->entries, 0, PtrackContentNblocks * sizeof(pg_atomic_uint64)); + /* + * Last part of memory representation of ptrack_map (crc) is actually unused + * so leave it as it is + */ + } } /* @@ -365,7 +316,6 @@ ptrackCheckpoint(void) /* Delete ptrack_map and all related files, if ptrack was switched off */ if (ptrack_map_size == 0) { - ptrackCleanFilesAndMap(); return; } else if (ptrack_map == NULL) @@ -392,9 +342,19 @@ ptrackCheckpoint(void) * into the memory with mmap after a crash/restart. That way, we have to * write values taking into account all paddings/alignments. * - * Write both magic and varsion_num at once. + * Write both magic and version_num at once. + */ + + /* + * Previously we read from the field magic, now we read from the beginning + * of the structure PtrackMapHdr. Make sure nothing has changed since then. */ - ptrack_write_chunk(ptrack_tmp_fd, &crc, (char *) &ptrack_map->magic, + StaticAssertStmt( + offsetof(PtrackMapHdr, magic) == 0, + "old write format for PtrackMapHdr.magic and PtrackMapHdr.version_num " + "is not upward-compatible"); + + ptrack_write_chunk(ptrack_tmp_fd, &crc, (char *) ptrack_map, offsetof(PtrackMapHdr, init_lsn)); init_lsn = pg_atomic_read_u64(&ptrack_map->init_lsn); @@ -509,20 +469,10 @@ assign_ptrack_map_size(int newval, void *extra) elog(DEBUG1, "assign_ptrack_map_size: MyProc %d newval %d ptrack_map_size " UINT64_FORMAT, MyProcPid, newval, ptrack_map_size); - /* - * XXX: for some reason assign_ptrack_map_size is called twice during the - * postmaster boot! First, it is always called with bootValue, so we use - * -1 as default value and no-op here. Next, it is called with the actual - * value from config. That way, we use 0 as an option for user to turn - * off ptrack and clean up all files. - */ - if (newval == -1) - return; - /* Delete ptrack_map and all related files, if ptrack was switched off. */ if (newval == 0) { - ptrackCleanFilesAndMap(); + ptrack_map_size = 0; return; } @@ -540,15 +490,6 @@ assign_ptrack_map_size(int newval, void *extra) elog(DEBUG1, "assign_ptrack_map_size: ptrack_map_size set to " UINT64_FORMAT, ptrack_map_size); - - /* Init map on postmaster start */ - if (!IsUnderPostmaster) - { - if (ptrack_map == NULL) - ptrackMapInit(); - } - else - ptrackMapAttach(); } } @@ -565,8 +506,13 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid, BlockNumber blkno, nblocks = 0; struct stat stat_buf; +#if PG_VERSION_NUM >= 170000 + RelFileNumber relNumber; + unsigned segno; +#else int oidchars; char oidbuf[OIDCHARS + 1]; +#endif /* Do not track temporary relations */ if (looks_like_temp_rel_name(filename)) @@ -575,22 +521,29 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid, /* Mark of non-temporary relation */ rnode.backend = InvalidBackendId; - rnode.node.dbNode = dbOid; - rnode.node.spcNode = tablespaceOid; + nodeDb(nodeOf(rnode)) = dbOid; + nodeSpc(nodeOf(rnode)) = tablespaceOid; + +#if PG_VERSION_NUM >= 170000 + if (!parse_filename_for_nontemp_relation(filename, &relNumber, &forknum, &segno)) + return; + nodeRel(nodeOf(rnode)) = relNumber; +#else if (!parse_filename_for_nontemp_relation(filename, &oidchars, &forknum)) return; memcpy(oidbuf, filename, oidchars); oidbuf[oidchars] = '\0'; - rnode.node.relNode = atooid(oidbuf); + nodeRel(nodeOf(rnode)) = atooid(oidbuf); +#endif /* Compute number of blocks based on file size */ if (stat(filepath, &stat_buf) == 0) nblocks = stat_buf.st_size / BLCKSZ; elog(DEBUG1, "ptrack_mark_file %s, nblocks %u rnode db %u spc %u rel %u, forknum %d", - filepath, nblocks, rnode.node.dbNode, rnode.node.spcNode, rnode.node.relNode, forknum); + filepath, nblocks, nodeDb(nodeOf(rnode)), nodeSpc(nodeOf(rnode)), nodeRel(nodeOf(rnode)), forknum); for (blkno = 0; blkno < nblocks; blkno++) ptrack_mark_block(rnode, forknum, blkno); @@ -640,12 +593,29 @@ ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid) } if (S_ISREG(fst.st_mode)) - ptrack_mark_file(dbOid, tablespaceOid, subpath, de->d_name); + ptrack_mark_file(dbOid, tablespaceOid, subpath, de->d_name); } FreeDir(dir); /* we ignore any error here */ } +static void +ptrack_atomic_increase(XLogRecPtr new_lsn, pg_atomic_uint64 *var) +{ + /* + * We use pg_atomic_uint64 here only for alignment purposes, because + * pg_atomic_uint64 is forcedly aligned on 8 bytes during the MSVC build. + */ + pg_atomic_uint64 old_lsn; + + old_lsn.value = pg_atomic_read_u64(var); +#if USE_ASSERT_CHECKING + elog(DEBUG3, "ptrack_mark_block: " UINT64_FORMAT " <- " UINT64_FORMAT, old_lsn.value, new_lsn); +#endif + while (old_lsn.value < new_lsn && + !pg_atomic_compare_exchange_u64(var, (uint64 *) &old_lsn.value, new_lsn)); +} + /* * Mark modified block in ptrack_map. */ @@ -655,15 +625,9 @@ ptrack_mark_block(RelFileNodeBackend smgr_rnode, { PtBlockId bid; uint64 hash; - size_t slot1; - size_t slot2; + size_t slots[2]; XLogRecPtr new_lsn; - /* - * We use pg_atomic_uint64 here only for alignment purposes, because - * pg_atomic_uint64 is forcedly aligned on 8 bytes during the MSVC build. - */ - pg_atomic_uint64 old_lsn; - pg_atomic_uint64 old_init_lsn; + int i; if (ptrack_map_size == 0 || ptrack_map == NULL @@ -671,13 +635,33 @@ ptrack_mark_block(RelFileNodeBackend smgr_rnode, * relations */ return; - bid.relnode = smgr_rnode.node; + bid.relnode = nodeOf(smgr_rnode); bid.forknum = forknum; bid.blocknum = blocknum; hash = BID_HASH_FUNC(bid); - slot1 = (size_t)(hash % PtrackContentNblocks); - slot2 = (size_t)(((hash << 32) | (hash >> 32)) % PtrackContentNblocks); + slots[0] = (size_t)(hash % PtrackContentNblocks); + slots[1] = (size_t)(((hash << 32) | (hash >> 32)) % PtrackContentNblocks); + + new_lsn = ptrack_set_init_lsn(); + + /* Atomically assign new LSN value to the slots */ + for (i = 0; i < lengthof(slots); i++) + { +#if USE_ASSERT_CHECKING + elog(DEBUG3, "ptrack_mark_block: map[%zu]", slots[i]); +#endif + ptrack_atomic_increase(new_lsn, &ptrack_map->entries[slots[i]]); + } +} + +XLogRecPtr +ptrack_set_init_lsn(void) +{ + XLogRecPtr new_lsn; + + if (ptrack_map_size == 0 || ptrack_map == NULL) + return InvalidXLogRecPtr; if (RecoveryInProgress()) new_lsn = GetXLogReplayRecPtr(NULL); @@ -685,24 +669,12 @@ ptrack_mark_block(RelFileNodeBackend smgr_rnode, new_lsn = GetXLogInsertRecPtr(); /* Atomically assign new init LSN value */ - old_init_lsn.value = pg_atomic_read_u64(&ptrack_map->init_lsn); - if (old_init_lsn.value == InvalidXLogRecPtr) + if (pg_atomic_read_u64(&ptrack_map->init_lsn) == InvalidXLogRecPtr) { - elog(DEBUG1, "ptrack_mark_block: init_lsn " UINT64_FORMAT " <- " UINT64_FORMAT, old_init_lsn.value, new_lsn); - - while (old_init_lsn.value < new_lsn && - !pg_atomic_compare_exchange_u64(&ptrack_map->init_lsn, (uint64 *) &old_init_lsn.value, new_lsn)); +#if USE_ASSERT_CHECKING + elog(DEBUG3, "ptrack_set_init_lsn: init_lsn"); +#endif + ptrack_atomic_increase(new_lsn, &ptrack_map->init_lsn); } - - /* Atomically assign new LSN value to the first slot */ - old_lsn.value = pg_atomic_read_u64(&ptrack_map->entries[slot1]); - elog(DEBUG3, "ptrack_mark_block: map[%zu]=" UINT64_FORMAT " <- " UINT64_FORMAT, slot1, old_lsn.value, new_lsn); - while (old_lsn.value < new_lsn && - !pg_atomic_compare_exchange_u64(&ptrack_map->entries[slot1], (uint64 *) &old_lsn.value, new_lsn)); - - /* And to the second */ - old_lsn.value = pg_atomic_read_u64(&ptrack_map->entries[slot2]); - elog(DEBUG3, "ptrack_mark_block: map[%zu]=" UINT64_FORMAT " <- " UINT64_FORMAT, slot2, old_lsn.value, new_lsn); - while (old_lsn.value < new_lsn && - !pg_atomic_compare_exchange_u64(&ptrack_map->entries[slot2], (uint64 *) &old_lsn.value, new_lsn)); + return new_lsn; } diff --git a/engine.h b/engine.h index 3386cc2..7ecddd2 100644 --- a/engine.h +++ b/engine.h @@ -4,6 +4,7 @@ * header for ptrack map for tracking updates of relation's pages * * + * Copyright (c) 2019-2022, Postgres Professional * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * @@ -23,9 +24,6 @@ /* #include "utils/relcache.h" */ #include "access/hash.h" - -/* Working copy of ptrack.map */ -#define PTRACK_MMAP_PATH "global/ptrack.map.mmap" /* Persistent copy of ptrack.map to restore after crash */ #define PTRACK_PATH "global/ptrack.map" /* Used for atomical crash-safe update of ptrack.map */ @@ -36,6 +34,9 @@ * buffer size for disk writes. On fast NVMe SSD it gives * around 20% increase in ptrack checkpoint speed compared * to PTRACK_BUF_SIZE == 1000, i.e. 8 KB writes. + * (PTRACK_BUS_SIZE is a count of pg_atomic_uint64) + * + * NOTE: but POSIX defines _POSIX_SSIZE_MAX as 32767 (bytes) */ #define PTRACK_BUF_SIZE ((uint64) 8000) @@ -80,7 +81,7 @@ typedef PtrackMapHdr * PtrackMap; #define PtrackActualSize \ (offsetof(PtrackMapHdr, entries) + PtrackContentNblocks * sizeof(pg_atomic_uint64) + sizeof(pg_crc32c)) -/* CRC32 value offset in order to directly access it in the mmap'ed memory chunk */ +/* CRC32 value offset in order to directly access it in the shared memory chunk */ #define PtrackCrcOffset (PtrackActualSize - sizeof(pg_crc32c)) /* Block address 'bid' to hash. To get slot position in map should be divided @@ -102,7 +103,8 @@ extern int ptrack_map_size_tmp; extern void ptrackCheckpoint(void); extern void ptrackMapInit(void); -extern void ptrackMapAttach(void); +extern void ptrackCleanFiles(void); +extern XLogRecPtr ptrack_set_init_lsn(void); extern void assign_ptrack_map_size(int newval, void *extra); diff --git a/make_dockerfile.sh b/make_dockerfile.sh deleted file mode 100755 index 52543e8..0000000 --- a/make_dockerfile.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env sh - -if [ -z ${PG_VERSION+x} ]; then - echo PG_VERSION is not set! - exit 1 -fi - -if [ -z ${PG_BRANCH+x} ]; then - echo PG_BRANCH is not set! - exit 1 -fi - -if [ -z ${MODE+x} ]; then - MODE=basic -else - echo MODE=${MODE} -fi - -if [ -z ${TEST_CASE+x} ]; then - TEST_CASE=all -else - echo TEST_CASE=${TEST_CASE} -fi - -if [ -z ${TEST_REPEATS+x} ]; then - TEST_REPEATS=1 -else - echo TEST_REPEATS=${TEST_REPEATS} -fi - -echo PG_VERSION=${PG_VERSION} -echo PG_BRANCH=${PG_BRANCH} - -sed \ - -e 's/${PG_VERSION}/'${PG_VERSION}/g \ - -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ - -e 's/${MODE}/'${MODE}/g \ - -e 's/${TEST_CASE}/'${TEST_CASE}/g \ - -e 's/${TEST_REPEATS}/'${TEST_REPEATS}/g \ -Dockerfile.in > Dockerfile diff --git a/patches/REL_11_STABLE-ptrack-core.diff b/patches/REL_11_STABLE-ptrack-core.diff index a8207f5..e78977c 100644 --- a/patches/REL_11_STABLE-ptrack-core.diff +++ b/patches/REL_11_STABLE-ptrack-core.diff @@ -207,24 +207,6 @@ index 80241455357..50dca7bf6f4 100644 #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 9a26295c8e8..dc72b27a10d 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 4fef3e21072..e55430879c3 100644 --- a/src/include/storage/copydir.h @@ -261,3 +243,16 @@ index 0298ed1a2bc..24c684771d0 100644 extern void mdinit(void); extern void mdclose(SMgrRelation reln, ForkNumber forknum); extern void mdcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo); +diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm +index b52baa90988..74870c048db 100644 +--- a/src/tools/msvc/Mkvcbuild.pm ++++ b/src/tools/msvc/Mkvcbuild.pm +@@ -33,7 +33,7 @@ my @unlink_on_exit; + # Set of variables for modules in contrib/ and src/test/modules/ + my $contrib_defines = { 'refint' => 'REFINT_VERBOSE' }; + my @contrib_uselibpq = ('dblink', 'oid2name', 'postgres_fdw', 'vacuumlo'); +-my @contrib_uselibpgport = ('oid2name', 'pg_standby', 'vacuumlo'); ++my @contrib_uselibpgport = ('oid2name', 'pg_standby', 'vacuumlo', 'ptrack'); + my @contrib_uselibpgcommon = ('oid2name', 'pg_standby', 'vacuumlo'); + my $contrib_extralibs = undef; + my $contrib_extraincludes = { 'dblink' => ['src/backend'] }; diff --git a/patches/REL_12_STABLE-ptrack-core.diff b/patches/REL_12_STABLE-ptrack-core.diff index d8c00e0..e3feb67 100644 --- a/patches/REL_12_STABLE-ptrack-core.diff +++ b/patches/REL_12_STABLE-ptrack-core.diff @@ -107,18 +107,6 @@ index aff3e885f36..4fffa5df17c 100644 /* Flag successful completion of ProcessSyncRequests */ sync_in_progress = false; } -diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index a70e79c4891..712f985f3e8 100644 ---- a/src/backend/utils/misc/guc.c -+++ b/src/backend/utils/misc/guc.c -@@ -581,7 +581,6 @@ static char *recovery_target_xid_string; - static char *recovery_target_name_string; - static char *recovery_target_lsn_string; - -- - /* should be static, but commands/variable.c needs to get at this */ - char *role_string; - diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 03c3da3d730..fdfe5c1318e 100644 --- a/src/bin/pg_checksums/pg_checksums.c @@ -237,24 +225,6 @@ index 61a24c2e3c6..cbd46d0cb02 100644 #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index fbd079d2439..01682035e0b 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 525cc6203e1..9481e1c5a88 100644 --- a/src/include/storage/copydir.h @@ -301,3 +271,16 @@ index 16428c5f5fb..6b0cd8f8eea 100644 extern void InitSync(void); extern void SyncPreCheckpoint(void); extern void SyncPostCheckpoint(void); +diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm +index 1bdc33d7168..83b1190775f 100644 +--- a/src/tools/msvc/Mkvcbuild.pm ++++ b/src/tools/msvc/Mkvcbuild.pm +@@ -33,7 +33,7 @@ my @unlink_on_exit; + # Set of variables for modules in contrib/ and src/test/modules/ + my $contrib_defines = { 'refint' => 'REFINT_VERBOSE' }; + my @contrib_uselibpq = ('dblink', 'oid2name', 'postgres_fdw', 'vacuumlo'); +-my @contrib_uselibpgport = ('oid2name', 'pg_standby', 'vacuumlo'); ++my @contrib_uselibpgport = ('oid2name', 'pg_standby', 'vacuumlo', 'ptrack'); + my @contrib_uselibpgcommon = ('oid2name', 'pg_standby', 'vacuumlo'); + my $contrib_extralibs = undef; + my $contrib_extraincludes = { 'dblink' => ['src/backend'] }; diff --git a/patches/REL_13_STABLE-ptrack-core.diff b/patches/REL_13_STABLE-ptrack-core.diff index 3491700..5b73162 100644 --- a/patches/REL_13_STABLE-ptrack-core.diff +++ b/patches/REL_13_STABLE-ptrack-core.diff @@ -1,9 +1,3 @@ -commit a14ac459d71528c64df00c693e9c71ac70d3ba29 -Author: anastasia -Date: Mon Oct 19 14:53:06 2020 +0300 - - add ptrack 2.0 - diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 50ae1f16d0..721b926ad2 100644 --- a/src/backend/replication/basebackup.c @@ -113,18 +107,6 @@ index 3ded2cdd71..3a596a59f7 100644 /* Flag successful completion of ProcessSyncRequests */ sync_in_progress = false; } -diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1683629ee3..d2fc154576 100644 ---- a/src/backend/utils/misc/guc.c -+++ b/src/backend/utils/misc/guc.c -@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; - static char *recovery_target_name_string; - static char *recovery_target_lsn_string; - -- - /* should be static, but commands/variable.c needs to get at this */ - char *role_string; - diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index ffdc23945c..7ae95866ce 100644 --- a/src/bin/pg_checksums/pg_checksums.c @@ -243,24 +225,6 @@ index 72e3352398..5c2e016501 100644 #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 3c6f906683..a7355f7ad1 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 5d28f59c1d..0d3f04d8af 100644 --- a/src/include/storage/copydir.h @@ -307,3 +271,16 @@ index e16ab8e711..88da9686eb 100644 extern void InitSync(void); extern void SyncPreCheckpoint(void); extern void SyncPostCheckpoint(void); +diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm +index 67b2ea9ee9b..e9a282d5647 100644 +--- a/src/tools/msvc/Mkvcbuild.pm ++++ b/src/tools/msvc/Mkvcbuild.pm +@@ -34,7 +34,7 @@ my @unlink_on_exit; + # Set of variables for modules in contrib/ and src/test/modules/ + my $contrib_defines = { 'refint' => 'REFINT_VERBOSE' }; + my @contrib_uselibpq = ('dblink', 'oid2name', 'postgres_fdw', 'vacuumlo'); +-my @contrib_uselibpgport = ('oid2name', 'pg_standby', 'vacuumlo'); ++my @contrib_uselibpgport = ('oid2name', 'pg_standby', 'vacuumlo', 'ptrack'); + my @contrib_uselibpgcommon = ('oid2name', 'pg_standby', 'vacuumlo'); + my $contrib_extralibs = undef; + my $contrib_extraincludes = { 'dblink' => ['src/backend'] }; diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff new file mode 100644 index 0000000..88ffcdc --- /dev/null +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -0,0 +1,286 @@ +diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c +index 50ae1f16d0..721b926ad2 100644 +--- a/src/backend/replication/basebackup.c ++++ b/src/backend/replication/basebackup.c +@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ /* ++ * Skip all transient ptrack files, but do copy ptrack.map, since it may ++ * be successfully used immediately after backup. TODO: check, test? ++ */ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c +index 0cf598dd0c..c9c44a4ae7 100644 +--- a/src/backend/storage/file/copydir.c ++++ b/src/backend/storage/file/copydir.c +@@ -27,6 +27,8 @@ + #include "storage/copydir.h" + #include "storage/fd.h" + ++copydir_hook_type copydir_hook = NULL; ++ + /* + * copydir: copy a directory + * +@@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) + } + FreeDir(xldir); + ++ if (copydir_hook) ++ copydir_hook(todir); ++ + /* + * Be paranoid here and fsync all files to ensure the copy is really done. + * But if fsync is disabled, we're done. +diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c +index 0eacd461cd..c2ef404a1a 100644 +--- a/src/backend/storage/smgr/md.c ++++ b/src/backend/storage/smgr/md.c +@@ -87,6 +87,8 @@ typedef struct _MdfdVec + + static MemoryContext MdCxt; /* context for all MdfdVec objects */ + ++mdextend_hook_type mdextend_hook = NULL; ++mdwrite_hook_type mdwrite_hook = NULL; + + /* Populate a file tag describing an md.c segment file. */ + #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ +@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + register_dirty_segment(reln, forknum, v); + + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); ++ ++ if (mdextend_hook) ++ mdextend_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + + if (!skipFsync && !SmgrIsTemp(reln)) + register_dirty_segment(reln, forknum, v); ++ ++ if (mdwrite_hook) ++ mdwrite_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c +index 3ded2cdd71..3a596a59f7 100644 +--- a/src/backend/storage/sync/sync.c ++++ b/src/backend/storage/sync/sync.c +@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ + static CycleCtr sync_cycle_ctr = 0; + static CycleCtr checkpoint_cycle_ctr = 0; + ++ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; ++ + /* Intervals for calling AbsorbSyncRequests */ + #define FSYNCS_PER_ABSORB 10 + #define UNLINKS_PER_ABSORB 10 +@@ -420,6 +422,9 @@ ProcessSyncRequests(void) + CheckpointStats.ckpt_longest_sync = longest; + CheckpointStats.ckpt_agg_sync_time = total_elapsed; + ++ if (ProcessSyncRequests_hook) ++ ProcessSyncRequests_hook(); ++ + /* Flag successful completion of ProcessSyncRequests */ + sync_in_progress = false; + } +diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c +index ffdc23945c..7ae95866ce 100644 +--- a/src/bin/pg_checksums/pg_checksums.c ++++ b/src/bin/pg_checksums/pg_checksums.c +@@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c +index 233441837f..cf7bd073bf 100644 +--- a/src/bin/pg_resetwal/pg_resetwal.c ++++ b/src/bin/pg_resetwal/pg_resetwal.c +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); + static void FindEndOfXLOG(void); + static void KillExistingXLOG(void); + static void KillExistingArchiveStatus(void); ++static void KillExistingPtrack(void); + static void WriteEmptyXLOG(void); + static void usage(void); + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) + RewriteControlFile(); + KillExistingXLOG(); + KillExistingArchiveStatus(); ++ KillExistingPtrack(); + WriteEmptyXLOG(); + + printf(_("Write-ahead log reset\n")); +@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) + } + } + ++/* ++ * Remove existing ptrack files ++ */ ++static void ++KillExistingPtrack(void) ++{ ++#define PTRACKDIR "global" ++ ++ DIR *xldir; ++ struct dirent *xlde; ++ char path[MAXPGPATH + sizeof(PTRACKDIR)]; ++ ++ xldir = opendir(PTRACKDIR); ++ if (xldir == NULL) ++ { ++ pg_log_error("could not open directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++ ++ while (errno = 0, (xlde = readdir(xldir)) != NULL) ++ { ++ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 || ++ strcmp(xlde->d_name, "ptrack.map") == 0 || ++ strcmp(xlde->d_name, "ptrack.map.tmp") == 0) ++ { ++ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name); ++ if (unlink(path) < 0) ++ { ++ pg_log_error("could not delete file \"%s\": %m", path); ++ exit(1); ++ } ++ } ++ } ++ ++ if (errno) ++ { ++ pg_log_error("could not read directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++ ++ if (closedir(xldir)) ++ { ++ pg_log_error("could not close directory \"%s\": %m", PTRACKDIR); ++ exit(1); ++ } ++} ++ + + /* + * Write an empty XLOG file, containing only the checkpoint record +diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c +index fbb97b5cf1..6cd7f2ae3e 100644 +--- a/src/bin/pg_rewind/filemap.c ++++ b/src/bin/pg_rewind/filemap.c +@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h +index 72e3352398..5c2e016501 100644 +--- a/src/include/miscadmin.h ++++ b/src/include/miscadmin.h +@@ -388,7 +388,7 @@ typedef enum ProcessingMode + NormalProcessing /* normal processing */ + } ProcessingMode; + +-extern ProcessingMode Mode; ++extern PGDLLIMPORT ProcessingMode Mode; + + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) + #define IsInitProcessingMode() (Mode == InitProcessing) +diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h +index 5d28f59c1d..0d3f04d8af 100644 +--- a/src/include/storage/copydir.h ++++ b/src/include/storage/copydir.h +@@ -13,6 +13,9 @@ + #ifndef COPYDIR_H + #define COPYDIR_H + ++typedef void (*copydir_hook_type) (const char *path); ++extern PGDLLIMPORT copydir_hook_type copydir_hook; ++ + extern void copydir(char *fromdir, char *todir, bool recurse); + extern void copy_file(char *fromfile, char *tofile); + +diff --git a/src/include/storage/md.h b/src/include/storage/md.h +index 07fd1bb7d0..5294811bc8 100644 +--- a/src/include/storage/md.h ++++ b/src/include/storage/md.h +@@ -19,6 +19,13 @@ + #include "storage/smgr.h" + #include "storage/sync.h" + ++typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdextend_hook_type mdextend_hook; ++typedef void (*mdwrite_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook; ++ + /* md storage manager functionality */ + extern void mdinit(void); + extern void mdopen(SMgrRelation reln); +diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h +index e16ab8e711..88da9686eb 100644 +--- a/src/include/storage/sync.h ++++ b/src/include/storage/sync.h +@@ -50,6 +50,9 @@ typedef struct FileTag + uint32 segno; + } FileTag; + ++typedef void (*ProcessSyncRequests_hook_type) (void); ++extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; ++ + extern void InitSync(void); + extern void SyncPreCheckpoint(void); + extern void SyncPostCheckpoint(void); +diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm +index 9b6539fb15d..4b2bcdb6b88 100644 +--- a/src/tools/msvc/Mkvcbuild.pm ++++ b/src/tools/msvc/Mkvcbuild.pm +@@ -38,7 +38,7 @@ my @unlink_on_exit; + my $contrib_defines = { 'refint' => 'REFINT_VERBOSE' }; + my @contrib_uselibpq = + ('dblink', 'oid2name', 'postgres_fdw', 'vacuumlo', 'libpq_pipeline'); +-my @contrib_uselibpgport = ('libpq_pipeline', 'oid2name', 'vacuumlo'); ++my @contrib_uselibpgport = ('libpq_pipeline', 'oid2name', 'vacuumlo', 'ptrack'); + my @contrib_uselibpgcommon = ('libpq_pipeline', 'oid2name', 'vacuumlo'); + my $contrib_extralibs = { 'libpq_pipeline' => ['ws2_32.lib'] }; + my $contrib_extraincludes = { 'dblink' => ['src/backend'] }; diff --git a/patches/REL_15_STABLE-ptrack-core.diff b/patches/REL_15_STABLE-ptrack-core.diff new file mode 100644 index 0000000..2adc5f3 --- /dev/null +++ b/patches/REL_15_STABLE-ptrack-core.diff @@ -0,0 +1,248 @@ +diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c +index cc16c4b331f..69b1af16cf5 100644 +--- a/src/backend/backup/basebackup.c ++++ b/src/backend/backup/basebackup.c +@@ -197,6 +197,13 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ /* ++ * Skip all transient ptrack files, but do copy ptrack.map, since it may ++ * be successfully used immediately after backup. TODO: check, test? ++ */ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +@@ -212,6 +219,11 @@ static const struct exclude_list_item noChecksumFiles[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c +index 658fd95ba95..eee38eba176 100644 +--- a/src/backend/storage/file/copydir.c ++++ b/src/backend/storage/file/copydir.c +@@ -27,6 +27,8 @@ + #include "storage/copydir.h" + #include "storage/fd.h" + ++copydir_hook_type copydir_hook = NULL; ++ + /* + * copydir: copy a directory + * +@@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) + } + FreeDir(xldir); + ++ if (copydir_hook) ++ copydir_hook(todir); ++ + /* + * Be paranoid here and fsync all files to ensure the copy is really done. + * But if fsync is disabled, we're done. +diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c +index a0fc60b32a3..7f091951c0d 100644 +--- a/src/backend/storage/smgr/md.c ++++ b/src/backend/storage/smgr/md.c +@@ -87,6 +87,8 @@ typedef struct _MdfdVec + + static MemoryContext MdCxt; /* context for all MdfdVec objects */ + ++mdextend_hook_type mdextend_hook = NULL; ++mdwrite_hook_type mdwrite_hook = NULL; + + /* Populate a file tag describing an md.c segment file. */ + #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ +@@ -484,6 +486,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + register_dirty_segment(reln, forknum, v); + + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); ++ ++ if (mdextend_hook) ++ mdextend_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +@@ -773,6 +778,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + + if (!skipFsync && !SmgrIsTemp(reln)) + register_dirty_segment(reln, forknum, v); ++ ++ if (mdwrite_hook) ++ mdwrite_hook(reln->smgr_rnode, forknum, blocknum); + } + + /* +diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c +index e1fb6310038..76d75680b31 100644 +--- a/src/backend/storage/sync/sync.c ++++ b/src/backend/storage/sync/sync.c +@@ -81,6 +81,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ + static CycleCtr sync_cycle_ctr = 0; + static CycleCtr checkpoint_cycle_ctr = 0; + ++ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; ++ + /* Intervals for calling AbsorbSyncRequests */ + #define FSYNCS_PER_ABSORB 10 + #define UNLINKS_PER_ABSORB 10 +@@ -477,6 +479,9 @@ ProcessSyncRequests(void) + CheckpointStats.ckpt_longest_sync = longest; + CheckpointStats.ckpt_agg_sync_time = total_elapsed; + ++ if (ProcessSyncRequests_hook) ++ ProcessSyncRequests_hook(); ++ + /* Flag successful completion of ProcessSyncRequests */ + sync_in_progress = false; + } +diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c +index 21dfe1b6ee5..266ac1ef40a 100644 +--- a/src/bin/pg_checksums/pg_checksums.c ++++ b/src/bin/pg_checksums/pg_checksums.c +@@ -118,6 +118,11 @@ static const struct exclude_list_item skip[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c +index d4772a29650..3318f64359d 100644 +--- a/src/bin/pg_resetwal/pg_resetwal.c ++++ b/src/bin/pg_resetwal/pg_resetwal.c +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); + static void FindEndOfXLOG(void); + static void KillExistingXLOG(void); + static void KillExistingArchiveStatus(void); ++static void KillExistingPtrack(void); + static void WriteEmptyXLOG(void); + static void usage(void); + +@@ -488,6 +489,7 @@ main(int argc, char *argv[]) + RewriteControlFile(); + KillExistingXLOG(); + KillExistingArchiveStatus(); ++ KillExistingPtrack(); + WriteEmptyXLOG(); + + printf(_("Write-ahead log reset\n")); +@@ -1036,6 +1038,41 @@ KillExistingArchiveStatus(void) + pg_fatal("could not close directory \"%s\": %m", ARCHSTATDIR); + } + ++/* ++ * Remove existing ptrack files ++ */ ++static void ++KillExistingPtrack(void) ++{ ++#define PTRACKDIR "global" ++ ++ DIR *xldir; ++ struct dirent *xlde; ++ char path[MAXPGPATH + sizeof(PTRACKDIR)]; ++ ++ xldir = opendir(PTRACKDIR); ++ if (xldir == NULL) ++ pg_fatal("could not open directory \"%s\": %m", PTRACKDIR); ++ ++ while (errno = 0, (xlde = readdir(xldir)) != NULL) ++ { ++ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 || ++ strcmp(xlde->d_name, "ptrack.map") == 0 || ++ strcmp(xlde->d_name, "ptrack.map.tmp") == 0) ++ { ++ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name); ++ if (unlink(path) < 0) ++ pg_fatal("could not delete file \"%s\": %m", path); ++ } ++ } ++ ++ if (errno) ++ pg_fatal("could not read directory \"%s\": %m", PTRACKDIR); ++ ++ if (closedir(xldir)) ++ pg_fatal("could not close directory \"%s\": %m", PTRACKDIR); ++} ++ + + /* + * Write an empty XLOG file, containing only the checkpoint record +diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c +index 62529310415..b496f54fb06 100644 +--- a/src/bin/pg_rewind/filemap.c ++++ b/src/bin/pg_rewind/filemap.c +@@ -157,6 +157,10 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h +index 50a26edeb06..af1602f5154 100644 +--- a/src/include/storage/copydir.h ++++ b/src/include/storage/copydir.h +@@ -13,6 +13,9 @@ + #ifndef COPYDIR_H + #define COPYDIR_H + ++typedef void (*copydir_hook_type) (const char *path); ++extern PGDLLIMPORT copydir_hook_type copydir_hook; ++ + extern void copydir(char *fromdir, char *todir, bool recurse); + extern void copy_file(char *fromfile, char *tofile); + +diff --git a/src/include/storage/md.h b/src/include/storage/md.h +index ffffa40db71..3ff98e0bf01 100644 +--- a/src/include/storage/md.h ++++ b/src/include/storage/md.h +@@ -19,6 +19,13 @@ + #include "storage/smgr.h" + #include "storage/sync.h" + ++typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdextend_hook_type mdextend_hook; ++typedef void (*mdwrite_hook_type) (RelFileNodeBackend smgr_rnode, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook; ++ + /* md storage manager functionality */ + extern void mdinit(void); + extern void mdopen(SMgrRelation reln); +diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h +index 9737e1eb67c..914ad86328f 100644 +--- a/src/include/storage/sync.h ++++ b/src/include/storage/sync.h +@@ -55,6 +55,9 @@ typedef struct FileTag + uint32 segno; + } FileTag; + ++typedef void (*ProcessSyncRequests_hook_type) (void); ++extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; ++ + extern void InitSync(void); + extern void SyncPreCheckpoint(void); + extern void SyncPostCheckpoint(void); diff --git a/patches/REL_16_STABLE-ptrack-core.diff b/patches/REL_16_STABLE-ptrack-core.diff new file mode 100644 index 0000000..04cf8a4 --- /dev/null +++ b/patches/REL_16_STABLE-ptrack-core.diff @@ -0,0 +1,261 @@ +diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c +index 45be21131c5..134e677f9d1 100644 +--- a/src/backend/backup/basebackup.c ++++ b/src/backend/backup/basebackup.c +@@ -199,6 +199,13 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ /* ++ * Skip all transient ptrack files, but do copy ptrack.map, since it may ++ * be successfully used immediately after backup. TODO: check, test? ++ */ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +@@ -214,6 +221,11 @@ static const struct exclude_list_item noChecksumFiles[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c +index e04bc3941ae..996b5de6169 100644 +--- a/src/backend/storage/file/copydir.c ++++ b/src/backend/storage/file/copydir.c +@@ -27,6 +27,8 @@ + #include "storage/copydir.h" + #include "storage/fd.h" + ++copydir_hook_type copydir_hook = NULL; ++ + /* + * copydir: copy a directory + * +@@ -75,6 +77,9 @@ copydir(const char *fromdir, const char *todir, bool recurse) + } + FreeDir(xldir); + ++ if (copydir_hook) ++ copydir_hook(todir); ++ + /* + * Be paranoid here and fsync all files to ensure the copy is really done. + * But if fsync is disabled, we're done. +diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c +index fdecbad1709..f849d00161e 100644 +--- a/src/backend/storage/smgr/md.c ++++ b/src/backend/storage/smgr/md.c +@@ -87,6 +87,8 @@ typedef struct _MdfdVec + + static MemoryContext MdCxt; /* context for all MdfdVec objects */ + ++mdextend_hook_type mdextend_hook = NULL; ++mdwrite_hook_type mdwrite_hook = NULL; + + /* Populate a file tag describing an md.c segment file. */ + #define INIT_MD_FILETAG(a,xx_rlocator,xx_forknum,xx_segno) \ +@@ -515,6 +517,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + register_dirty_segment(reln, forknum, v); + + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); ++ ++ if (mdextend_hook) ++ mdextend_hook(reln->smgr_rlocator, forknum, blocknum); + } + + /* +@@ -622,6 +627,12 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, + + remblocks -= numblocks; + curblocknum += numblocks; ++ ++ if (mdextend_hook) ++ { ++ for (; blocknum < curblocknum; blocknum++) ++ mdextend_hook(reln->smgr_rlocator, forknum, blocknum); ++ } + } + } + +@@ -867,6 +878,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + + if (!skipFsync && !SmgrIsTemp(reln)) + register_dirty_segment(reln, forknum, v); ++ ++ if (mdwrite_hook) ++ mdwrite_hook(reln->smgr_rlocator, forknum, blocknum); + } + + /* +diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c +index 04fcb06056d..22bf179f560 100644 +--- a/src/backend/storage/sync/sync.c ++++ b/src/backend/storage/sync/sync.c +@@ -79,6 +79,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ + static CycleCtr sync_cycle_ctr = 0; + static CycleCtr checkpoint_cycle_ctr = 0; + ++ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; ++ + /* Intervals for calling AbsorbSyncRequests */ + #define FSYNCS_PER_ABSORB 10 + #define UNLINKS_PER_ABSORB 10 +@@ -475,6 +477,9 @@ ProcessSyncRequests(void) + CheckpointStats.ckpt_longest_sync = longest; + CheckpointStats.ckpt_agg_sync_time = total_elapsed; + ++ if (ProcessSyncRequests_hook) ++ ProcessSyncRequests_hook(); ++ + /* Flag successful completion of ProcessSyncRequests */ + sync_in_progress = false; + } +diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c +index 19eb67e4854..008a7acc9f0 100644 +--- a/src/bin/pg_checksums/pg_checksums.c ++++ b/src/bin/pg_checksums/pg_checksums.c +@@ -118,6 +118,11 @@ static const struct exclude_list_item skip[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c +index e7ef2b8bd0c..ca7f8cdbc2f 100644 +--- a/src/bin/pg_resetwal/pg_resetwal.c ++++ b/src/bin/pg_resetwal/pg_resetwal.c +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); + static void FindEndOfXLOG(void); + static void KillExistingXLOG(void); + static void KillExistingArchiveStatus(void); ++static void KillExistingPtrack(void); + static void WriteEmptyXLOG(void); + static void usage(void); + +@@ -488,6 +489,7 @@ main(int argc, char *argv[]) + RewriteControlFile(); + KillExistingXLOG(); + KillExistingArchiveStatus(); ++ KillExistingPtrack(); + WriteEmptyXLOG(); + + printf(_("Write-ahead log reset\n")); +@@ -1029,6 +1031,41 @@ KillExistingArchiveStatus(void) + pg_fatal("could not close directory \"%s\": %m", ARCHSTATDIR); + } + ++/* ++ * Remove existing ptrack files ++ */ ++static void ++KillExistingPtrack(void) ++{ ++#define PTRACKDIR "global" ++ ++ DIR *xldir; ++ struct dirent *xlde; ++ char path[MAXPGPATH + sizeof(PTRACKDIR)]; ++ ++ xldir = opendir(PTRACKDIR); ++ if (xldir == NULL) ++ pg_fatal("could not open directory \"%s\": %m", PTRACKDIR); ++ ++ while (errno = 0, (xlde = readdir(xldir)) != NULL) ++ { ++ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 || ++ strcmp(xlde->d_name, "ptrack.map") == 0 || ++ strcmp(xlde->d_name, "ptrack.map.tmp") == 0) ++ { ++ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name); ++ if (unlink(path) < 0) ++ pg_fatal("could not delete file \"%s\": %m", path); ++ } ++ } ++ ++ if (errno) ++ pg_fatal("could not read directory \"%s\": %m", PTRACKDIR); ++ ++ if (closedir(xldir)) ++ pg_fatal("could not close directory \"%s\": %m", PTRACKDIR); ++} ++ + + /* + * Write an empty XLOG file, containing only the checkpoint record +diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c +index bd5c598e200..a568156c5fb 100644 +--- a/src/bin/pg_rewind/filemap.c ++++ b/src/bin/pg_rewind/filemap.c +@@ -157,6 +157,10 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h +index a8be5b21e0b..020874f96cd 100644 +--- a/src/include/storage/copydir.h ++++ b/src/include/storage/copydir.h +@@ -13,6 +13,9 @@ + #ifndef COPYDIR_H + #define COPYDIR_H + ++typedef void (*copydir_hook_type) (const char *path); ++extern PGDLLIMPORT copydir_hook_type copydir_hook; ++ + extern void copydir(const char *fromdir, const char *todir, bool recurse); + extern void copy_file(const char *fromfile, const char *tofile); + +diff --git a/src/include/storage/md.h b/src/include/storage/md.h +index 941879ee6a8..24738aeecd0 100644 +--- a/src/include/storage/md.h ++++ b/src/include/storage/md.h +@@ -19,6 +19,13 @@ + #include "storage/smgr.h" + #include "storage/sync.h" + ++typedef void (*mdextend_hook_type) (RelFileLocatorBackend smgr_rlocator, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdextend_hook_type mdextend_hook; ++typedef void (*mdwrite_hook_type) (RelFileLocatorBackend smgr_rlocator, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook; ++ + /* md storage manager functionality */ + extern void mdinit(void); + extern void mdopen(SMgrRelation reln); +diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h +index cfbcfa6797d..2a432440db9 100644 +--- a/src/include/storage/sync.h ++++ b/src/include/storage/sync.h +@@ -55,6 +55,9 @@ typedef struct FileTag + uint32 segno; + } FileTag; + ++typedef void (*ProcessSyncRequests_hook_type) (void); ++extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; ++ + extern void InitSync(void); + extern void SyncPreCheckpoint(void); + extern void SyncPostCheckpoint(void); diff --git a/patches/master-ptrack-core.diff b/patches/master-ptrack-core.diff new file mode 100644 index 0000000..3357a2b --- /dev/null +++ b/patches/master-ptrack-core.diff @@ -0,0 +1,294 @@ +diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c +index 34a2c71812..2d73d8023e 100644 +--- a/src/backend/access/transam/xlog.c ++++ b/src/backend/access/transam/xlog.c +@@ -135,6 +135,7 @@ int wal_retrieve_retry_interval = 5000; + int max_slot_wal_keep_size_mb = -1; + int wal_decode_buffer_size = 512 * 1024; + bool track_wal_io_timing = false; ++backup_checkpoint_request_hook_type backup_checkpoint_request_hook = NULL; + + #ifdef WAL_DEBUG + bool XLOG_DEBUG = false; +@@ -8801,6 +8802,12 @@ do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, + { + bool checkpointfpw; + ++ /* ++ * Before we call RequestCheckpoint() we need to set ++ * init_lsn for ptrack map ++ */ ++ if (backup_checkpoint_request_hook) ++ backup_checkpoint_request_hook(); + /* + * Force a CHECKPOINT. Aside from being necessary to prevent torn + * page problems, this guarantees that two successive backup runs +diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c +index 9a2bf59e84..ade9115651 100644 +--- a/src/backend/backup/basebackup.c ++++ b/src/backend/backup/basebackup.c +@@ -220,6 +220,13 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ /* ++ * Skip all transient ptrack files, but do copy ptrack.map, since it may ++ * be successfully used immediately after backup. TODO: check, test? ++ */ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c +index d4fbe54207..b108416c70 100644 +--- a/src/backend/storage/file/copydir.c ++++ b/src/backend/storage/file/copydir.c +@@ -27,6 +27,8 @@ + #include "storage/copydir.h" + #include "storage/fd.h" + ++copydir_hook_type copydir_hook = NULL; ++ + /* + * copydir: copy a directory + * +@@ -75,6 +77,9 @@ copydir(const char *fromdir, const char *todir, bool recurse) + } + FreeDir(xldir); + ++ if (copydir_hook) ++ copydir_hook(todir); ++ + /* + * Be paranoid here and fsync all files to ensure the copy is really done. + * But if fsync is disabled, we're done. +diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c +index bf0f3ca76d..7d9833a360 100644 +--- a/src/backend/storage/smgr/md.c ++++ b/src/backend/storage/smgr/md.c +@@ -85,6 +85,8 @@ typedef struct _MdfdVec + + static MemoryContext MdCxt; /* context for all MdfdVec objects */ + ++mdextend_hook_type mdextend_hook = NULL; ++mdwrite_hook_type mdwrite_hook = NULL; + + /* Populate a file tag describing an md.c segment file. */ + #define INIT_MD_FILETAG(a,xx_rlocator,xx_forknum,xx_segno) \ +@@ -513,6 +515,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + register_dirty_segment(reln, forknum, v); + + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); ++ ++ if (mdextend_hook) ++ mdextend_hook(reln->smgr_rlocator, forknum, blocknum); + } + + /* +@@ -620,6 +625,12 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, + + remblocks -= numblocks; + curblocknum += numblocks; ++ ++ if (mdextend_hook) ++ { ++ for (; blocknum < curblocknum; blocknum++) ++ mdextend_hook(reln->smgr_rlocator, forknum, blocknum); ++ } + } + } + +@@ -1015,7 +1026,14 @@ mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + + nblocks -= nblocks_this_segment; + buffers += nblocks_this_segment; +- blocknum += nblocks_this_segment; ++ ++ if (mdwrite_hook) ++ { ++ for (; nblocks_this_segment--; blocknum++) ++ mdwrite_hook(reln->smgr_rlocator, forknum, blocknum); ++ } ++ else ++ blocknum += nblocks_this_segment; + } + } + +diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c +index ab7137d0ff..bc40a763c0 100644 +--- a/src/backend/storage/sync/sync.c ++++ b/src/backend/storage/sync/sync.c +@@ -74,6 +74,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ + static CycleCtr sync_cycle_ctr = 0; + static CycleCtr checkpoint_cycle_ctr = 0; + ++ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; ++ + /* Intervals for calling AbsorbSyncRequests */ + #define FSYNCS_PER_ABSORB 10 + #define UNLINKS_PER_ABSORB 10 +@@ -470,6 +472,9 @@ ProcessSyncRequests(void) + CheckpointStats.ckpt_longest_sync = longest; + CheckpointStats.ckpt_agg_sync_time = total_elapsed; + ++ if (ProcessSyncRequests_hook) ++ ProcessSyncRequests_hook(); ++ + /* Flag successful completion of ProcessSyncRequests */ + sync_in_progress = false; + } +diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c +index 9e6fd435f6..f2180b9f6d 100644 +--- a/src/bin/pg_checksums/pg_checksums.c ++++ b/src/bin/pg_checksums/pg_checksums.c +@@ -110,6 +110,11 @@ static const struct exclude_list_item skip[] = { + {"pg_filenode.map", false}, + {"pg_internal.init", true}, + {"PG_VERSION", false}, ++ ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + #ifdef EXEC_BACKEND + {"config_exec_params", true}, + #endif +diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c +index e9dcb5a6d8..844b04d5e1 100644 +--- a/src/bin/pg_resetwal/pg_resetwal.c ++++ b/src/bin/pg_resetwal/pg_resetwal.c +@@ -86,6 +86,7 @@ static void FindEndOfXLOG(void); + static void KillExistingXLOG(void); + static void KillExistingArchiveStatus(void); + static void KillExistingWALSummaries(void); ++static void KillExistingPtrack(void); + static void WriteEmptyXLOG(void); + static void usage(void); + +@@ -495,6 +496,7 @@ main(int argc, char *argv[]) + KillExistingXLOG(); + KillExistingArchiveStatus(); + KillExistingWALSummaries(); ++ KillExistingPtrack(); + WriteEmptyXLOG(); + + printf(_("Write-ahead log reset\n")); +@@ -998,6 +1000,41 @@ KillExistingXLOG(void) + pg_fatal("could not close directory \"%s\": %m", XLOGDIR); + } + ++/* ++ * Remove existing ptrack files ++ */ ++static void ++KillExistingPtrack(void) ++{ ++#define PTRACKDIR "global" ++ ++ DIR *xldir; ++ struct dirent *xlde; ++ char path[MAXPGPATH + sizeof(PTRACKDIR)]; ++ ++ xldir = opendir(PTRACKDIR); ++ if (xldir == NULL) ++ pg_fatal("could not open directory \"%s\": %m", PTRACKDIR); ++ ++ while (errno = 0, (xlde = readdir(xldir)) != NULL) ++ { ++ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 || ++ strcmp(xlde->d_name, "ptrack.map") == 0 || ++ strcmp(xlde->d_name, "ptrack.map.tmp") == 0) ++ { ++ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name); ++ if (unlink(path) < 0) ++ pg_fatal("could not delete file \"%s\": %m", path); ++ } ++ } ++ ++ if (errno) ++ pg_fatal("could not read directory \"%s\": %m", PTRACKDIR); ++ ++ if (closedir(xldir)) ++ pg_fatal("could not close directory \"%s\": %m", PTRACKDIR); ++} ++ + + /* + * Remove existing archive status files +diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c +index 4458324c9d..7d857467f7 100644 +--- a/src/bin/pg_rewind/filemap.c ++++ b/src/bin/pg_rewind/filemap.c +@@ -156,6 +156,10 @@ static const struct exclude_list_item excludeFiles[] = + {"postmaster.pid", false}, + {"postmaster.opts", false}, + ++ {"ptrack.map.mmap", false}, ++ {"ptrack.map", false}, ++ {"ptrack.map.tmp", false}, ++ + /* end of list */ + {NULL, false} + }; +diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h +index 76787a8267..2c662f4022 100644 +--- a/src/include/access/xlog.h ++++ b/src/include/access/xlog.h +@@ -57,6 +57,9 @@ extern PGDLLIMPORT int wal_decode_buffer_size; + + extern PGDLLIMPORT int CheckPointSegments; + ++typedef void (*backup_checkpoint_request_hook_type) (void); ++extern PGDLLIMPORT backup_checkpoint_request_hook_type backup_checkpoint_request_hook; ++ + /* Archive modes */ + typedef enum ArchiveMode + { +diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h +index a25e258f47..b20b9c76e8 100644 +--- a/src/include/storage/copydir.h ++++ b/src/include/storage/copydir.h +@@ -13,6 +13,9 @@ + #ifndef COPYDIR_H + #define COPYDIR_H + ++typedef void (*copydir_hook_type) (const char *path); ++extern PGDLLIMPORT copydir_hook_type copydir_hook; ++ + extern void copydir(const char *fromdir, const char *todir, bool recurse); + extern void copy_file(const char *fromfile, const char *tofile); + +diff --git a/src/include/storage/md.h b/src/include/storage/md.h +index 620f10abde..b36936871b 100644 +--- a/src/include/storage/md.h ++++ b/src/include/storage/md.h +@@ -19,6 +19,13 @@ + #include "storage/smgr.h" + #include "storage/sync.h" + ++typedef void (*mdextend_hook_type) (RelFileLocatorBackend smgr_rlocator, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdextend_hook_type mdextend_hook; ++typedef void (*mdwrite_hook_type) (RelFileLocatorBackend smgr_rlocator, ++ ForkNumber forknum, BlockNumber blocknum); ++extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook; ++ + /* md storage manager functionality */ + extern void mdinit(void); + extern void mdopen(SMgrRelation reln); +diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h +index 9dee8fa6e5..348ed53e4e 100644 +--- a/src/include/storage/sync.h ++++ b/src/include/storage/sync.h +@@ -55,6 +55,9 @@ typedef struct FileTag + uint64 segno; + } FileTag; + ++typedef void (*ProcessSyncRequests_hook_type) (void); ++extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; ++ + extern void InitSync(void); + extern void SyncPreCheckpoint(void); + extern void SyncPostCheckpoint(void); diff --git a/ptrack--2.1--2.2.sql b/ptrack--2.1--2.2.sql index b09c15e..da897b6 100644 --- a/ptrack--2.1--2.2.sql +++ b/ptrack--2.1--2.2.sql @@ -1,7 +1,7 @@ /* ptrack/ptrack--2.1--2.2.sql */ -- Complain if script is sourced in psql, rather than via ALTER EXTENSION -\echo Use "ALTER EXTENSION ptrack UPDATE;" to load this file.\ quit +\echo Use "ALTER EXTENSION ptrack UPDATE;" to load this file. \quit DROP FUNCTION ptrack_get_pagemapset(start_lsn pg_lsn); CREATE FUNCTION ptrack_get_pagemapset(start_lsn pg_lsn) diff --git a/ptrack--2.2--2.3.sql b/ptrack--2.2--2.3.sql new file mode 100644 index 0000000..6c5f574 --- /dev/null +++ b/ptrack--2.2--2.3.sql @@ -0,0 +1,5 @@ +/* ptrack/ptrack--2.2--2.3.sql */ + +-- Complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION ptrack UPDATE;" to load this file. \quit + diff --git a/ptrack--2.3--2.4.sql b/ptrack--2.3--2.4.sql new file mode 100644 index 0000000..780bba5 --- /dev/null +++ b/ptrack--2.3--2.4.sql @@ -0,0 +1,5 @@ +/* ptrack/ptrack--2.3--2.4.sql */ + +-- Complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION ptrack UPDATE;" to load this file. \quit + diff --git a/ptrack.c b/ptrack.c index 66f5676..e2f3627 100644 --- a/ptrack.c +++ b/ptrack.c @@ -2,14 +2,13 @@ * ptrack.c * Block level incremental backup engine * - * Copyright (c) 2019-2020, Postgres Professional + * Copyright (c) 2019-2022, Postgres Professional * * IDENTIFICATION * ptrack/ptrack.c * * INTERFACE ROUTINES (PostgreSQL side) * ptrackMapInit() --- allocate new shared ptrack_map - * ptrackMapAttach() --- attach to the existing ptrack_map * assign_ptrack_map_size() --- ptrack_map_size GUC assign callback * ptrack_walkdir() --- walk directory and mark all blocks of all * data files in ptrack_map @@ -17,7 +16,7 @@ * * Currently ptrack has following public API methods: * - * # ptrack_version --- returns ptrack version string (2.0 currently). + * # ptrack_version --- returns ptrack version string (2.4 currently). * # ptrack_get_pagemapset('LSN') --- returns a set of changed data files with * bitmaps of changed blocks since specified LSN. * # ptrack_init_lsn --- returns LSN of the last ptrack map initialization. @@ -37,11 +36,9 @@ #include "funcapi.h" #include "miscadmin.h" #include "nodes/pg_list.h" -#ifdef PGPRO_EE -/* For file_is_in_cfs_tablespace() only. */ -#include "replication/basebackup.h" -#endif +#include "port/pg_crc32c.h" #include "storage/copydir.h" +#include "storage/ipc.h" #include "storage/lmgr.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -53,32 +50,43 @@ #include "utils/pg_lsn.h" #include "datapagemap.h" -#include "engine.h" #include "ptrack.h" +#include "engine.h" PG_MODULE_MAGIC; PtrackMap ptrack_map = NULL; -uint64 ptrack_map_size; +uint64 ptrack_map_size = 0; int ptrack_map_size_tmp; +static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static copydir_hook_type prev_copydir_hook = NULL; static mdwrite_hook_type prev_mdwrite_hook = NULL; static mdextend_hook_type prev_mdextend_hook = NULL; static ProcessSyncRequests_hook_type prev_ProcessSyncRequests_hook = NULL; +#if PG_VERSION_NUM >= 170000 +static backup_checkpoint_request_hook_type prev_backup_checkpoint_request_hook = NULL; +#endif void _PG_init(void); -void _PG_fini(void); +static void ptrack_shmem_startup_hook(void); static void ptrack_copydir_hook(const char *path); static void ptrack_mdwrite_hook(RelFileNodeBackend smgr_rnode, ForkNumber forkno, BlockNumber blkno); static void ptrack_mdextend_hook(RelFileNodeBackend smgr_rnode, ForkNumber forkno, BlockNumber blkno); static void ptrack_ProcessSyncRequests_hook(void); +#if PG_VERSION_NUM >= 170000 +static void ptrack_backup_checkpoint_request_hook(void); +#endif static void ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid); static int ptrack_filelist_getnext(PtScanCtx * ctx); +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void ptrack_shmem_request(void); +#endif /* * Module load callback @@ -103,15 +111,34 @@ _PG_init(void) "Sets the size of ptrack map in MB used for incremental backup (0 disabled).", NULL, &ptrack_map_size_tmp, - -1, - -1, 32 * 1024, /* limit to 32 GB */ - PGC_POSTMASTER, 0, +#if SIZEOF_SIZE_T == 8 + 0, 32 * 1024, /* limit to 32 GB */ +#else + 0, 256, /* limit to 256 MB */ +#endif + PGC_POSTMASTER, + GUC_UNIT_MB, NULL, assign_ptrack_map_size, NULL); + /* Request server shared memory */ + if (ptrack_map_size != 0) + { +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = ptrack_shmem_request; +#else + RequestAddinShmemSpace(PtrackActualSize); +#endif + } + else + ptrackCleanFiles(); + /* Install hooks */ + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = ptrack_shmem_startup_hook; prev_copydir_hook = copydir_hook; copydir_hook = ptrack_copydir_hook; prev_mdwrite_hook = mdwrite_hook; @@ -120,19 +147,56 @@ _PG_init(void) mdextend_hook = ptrack_mdextend_hook; prev_ProcessSyncRequests_hook = ProcessSyncRequests_hook; ProcessSyncRequests_hook = ptrack_ProcessSyncRequests_hook; +#if PG_VERSION_NUM >= 170000 + prev_backup_checkpoint_request_hook = backup_checkpoint_request_hook; + backup_checkpoint_request_hook = ptrack_backup_checkpoint_request_hook; +#endif } +#if PG_VERSION_NUM >= 150000 +static void +ptrack_shmem_request(void) +{ + if (prev_shmem_request_hook) + prev_shmem_request_hook(); + + RequestAddinShmemSpace(PtrackActualSize); +} +#endif + /* - * Module unload callback + * ptrack_shmem_startup hook: allocate or attach to shared memory. */ -void -_PG_fini(void) +static void +ptrack_shmem_startup_hook(void) { - /* Uninstall hooks */ - copydir_hook = prev_copydir_hook; - mdwrite_hook = prev_mdwrite_hook; - mdextend_hook = prev_mdextend_hook; - ProcessSyncRequests_hook = prev_ProcessSyncRequests_hook; + bool map_found; + + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + /* + * Create or attach to the shared memory state + */ + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + + if (ptrack_map_size != 0) + { + ptrack_map = ShmemInitStruct("ptrack map", + PtrackActualSize, + &map_found); + if (!map_found) + { + ptrackMapInit(); + elog(DEBUG1, "Shared memory for ptrack is ready"); + } + } + else + { + ptrack_map = NULL; + } + + LWLockRelease(AddinShmemInitLock); } /* @@ -178,14 +242,6 @@ ptrack_copydir_hook(const char *path) elog(DEBUG1, "ptrack_copydir_hook: spcOid %u, dbOid %u", spcOid, dbOid); -#ifdef PGPRO_EE - /* - * Currently, we do not track files from compressed tablespaces in ptrack. - */ - if (file_is_in_cfs_tablespace(path)) - elog(DEBUG1, "ptrack_copydir_hook: skipping changes tracking in the CFS tablespace %u", spcOid); - else -#endif ptrack_walkdir(path, spcOid, dbOid); if (prev_copydir_hook) @@ -221,6 +277,16 @@ ptrack_ProcessSyncRequests_hook() prev_ProcessSyncRequests_hook(); } +#if PG_VERSION_NUM >= 170000 +static void +ptrack_backup_checkpoint_request_hook(void) +{ + ptrack_set_init_lsn(); + + if (prev_backup_checkpoint_request_hook) + prev_backup_checkpoint_request_hook(); +} +#endif /* * Recursively walk through the path and add all data files to filelist. */ @@ -229,7 +295,6 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid) { DIR *dir; struct dirent *de; - dir = AllocateDir(path); while ((de = ReadDirExtended(dir, path, LOG)) != NULL) @@ -251,7 +316,7 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid) if (sret < 0) { - ereport(LOG, + ereport(WARNING, (errcode_for_file_access(), errmsg("ptrack: could not stat file \"%s\": %m", subpath))); continue; @@ -259,41 +324,58 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid) if (S_ISREG(fst.st_mode)) { + if (fst.st_size == 0) + { + elog(DEBUG3, "ptrack: skip empty file %s", subpath); + + /* But try the next one */ + continue; + } + /* Regular file inside database directory, otherwise skip it */ if (dbOid != InvalidOid || spcOid == GLOBALTABLESPACE_OID) { +#if PG_VERSION_NUM >= 170000 + RelFileNumber relNumber; + unsigned segno; +#else int oidchars; char oidbuf[OIDCHARS + 1]; +#endif char *segpath; PtrackFileList_i *pfl = palloc0(sizeof(PtrackFileList_i)); /* * Check that filename seems to be a regular relation file. */ +#if PG_VERSION_NUM >= 170000 + if (!parse_filename_for_nontemp_relation(de->d_name, &relNumber, &pfl->forknum, &segno)) + continue; +#else if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars, &pfl->forknum)) continue; +#endif + /* Parse segno */ + segpath = strstr(de->d_name, "."); + pfl->segno = segpath != NULL ? atoi(segpath + 1) : 0; - /* Parse segno for main fork */ - if (pfl->forknum == MAIN_FORKNUM) - { - segpath = strstr(de->d_name, "."); - pfl->segno = segpath != NULL ? atoi(segpath + 1) : 0; - } - else - pfl->segno = 0; - + /* Fill the pfl in */ +#if PG_VERSION_NUM >= 170000 + nodeRel(pfl->relnode) = relNumber; +#else memcpy(oidbuf, de->d_name, oidchars); oidbuf[oidchars] = '\0'; - pfl->relnode.relNode = atooid(oidbuf); - pfl->relnode.dbNode = dbOid; - pfl->relnode.spcNode = spcOid == InvalidOid ? DEFAULTTABLESPACE_OID : spcOid; - pfl->path = GetRelationPath(dbOid, pfl->relnode.spcNode, - pfl->relnode.relNode, InvalidBackendId, pfl->forknum); + nodeRel(pfl->relnode) = atooid(oidbuf); +#endif + nodeDb(pfl->relnode) = dbOid; + nodeSpc(pfl->relnode) = spcOid == InvalidOid ? DEFAULTTABLESPACE_OID : spcOid; + pfl->path = GetRelationPath(dbOid, nodeSpc(pfl->relnode), + nodeRel(pfl->relnode), InvalidBackendId, pfl->forknum); *filelist = lappend(*filelist, pfl); elog(DEBUG3, "ptrack: added file %s of rel %u to file list", - pfl->path, pfl->relnode.relNode); + pfl->path, nodeRel(pfl->relnode)); } } else if (S_ISDIR(fst.st_mode)) @@ -305,7 +387,7 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid) ptrack_gather_filelist(filelist, subpath, spcOid, InvalidOid); } /* TODO: is it enough to properly check symlink support? */ -#ifndef WIN32 +#if !defined(WIN32) || (PG_VERSION_NUM >= 160000) else if (S_ISLNK(fst.st_mode)) #else else if (pgwin32_is_junction(subpath)) @@ -330,17 +412,29 @@ ptrack_filelist_getnext(PtScanCtx * ctx) ListCell *cell; char *fullpath; struct stat fst; + uint32 rel_st_size = 0; + +get_next: /* No more file in the list */ if (list_length(ctx->filelist) == 0) return -1; +#ifdef foreach_current_index + /* Get first file from the head */ + cell = list_tail(ctx->filelist); + pfl = (PtrackFileList_i *) lfirst(cell); + + /* Remove this file from the list */ + ctx->filelist = list_delete_last(ctx->filelist); +#else /* Get first file from the head */ cell = list_head(ctx->filelist); pfl = (PtrackFileList_i *) lfirst(cell); /* Remove this file from the list */ ctx->filelist = list_delete_first(ctx->filelist); +#endif if (pfl->segno > 0) { @@ -354,9 +448,9 @@ ptrack_filelist_getnext(PtScanCtx * ctx) ctx->relpath = pfl->path; } - ctx->bid.relnode.spcNode = pfl->relnode.spcNode; - ctx->bid.relnode.dbNode = pfl->relnode.dbNode; - ctx->bid.relnode.relNode = pfl->relnode.relNode; + nodeSpc(ctx->bid.relnode) = nodeSpc(pfl->relnode); + nodeDb(ctx->bid.relnode) = nodeDb(pfl->relnode); + nodeRel(ctx->bid.relnode) = nodeRel(pfl->relnode); ctx->bid.forknum = pfl->forknum; ctx->bid.blocknum = 0; @@ -365,17 +459,27 @@ ptrack_filelist_getnext(PtScanCtx * ctx) elog(WARNING, "ptrack: cannot stat file %s", fullpath); /* But try the next one */ - return ptrack_filelist_getnext(ctx); + goto get_next; + } + + rel_st_size = fst.st_size; + + if (rel_st_size == 0) + { + elog(DEBUG3, "ptrack: skip empty file %s", fullpath); + + /* But try the next one */ + goto get_next; } if (pfl->segno > 0) { - ctx->relsize = pfl->segno * RELSEG_SIZE + fst.st_size / BLCKSZ; + ctx->relsize = pfl->segno * RELSEG_SIZE + rel_st_size / BLCKSZ; ctx->bid.blocknum = pfl->segno * RELSEG_SIZE; } else /* Estimate relsize as size of first segment in blocks */ - ctx->relsize = fst.st_size / BLCKSZ; + ctx->relsize = rel_st_size / BLCKSZ; elog(DEBUG3, "ptrack: got file %s with size %u from the file list", pfl->path, ctx->relsize); @@ -494,7 +598,7 @@ ptrack_get_pagemapset(PG_FUNCTION_ARGS) XLogRecPtr update_lsn2; /* Stop traversal if there are no more segments */ - if (ctx->bid.blocknum > ctx->relsize) + if (ctx->bid.blocknum >= ctx->relsize) { /* We completed a segment and there is a bitmap to return */ if (pagemap.bitmap != NULL) @@ -526,12 +630,9 @@ ptrack_get_pagemapset(PG_FUNCTION_ARGS) if (htup) SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(htup)); } - else - { - /* We have just processed unchanged file, let's pick next */ - if (ptrack_filelist_getnext(ctx) < 0) - SRF_RETURN_DONE(funcctx); - } + + if (ptrack_filelist_getnext(ctx) < 0) + SRF_RETURN_DONE(funcctx); } hash = BID_HASH_FUNC(ctx->bid); @@ -539,10 +640,12 @@ ptrack_get_pagemapset(PG_FUNCTION_ARGS) update_lsn1 = pg_atomic_read_u64(&ptrack_map->entries[slot1]); +#if USE_ASSERT_CHECKING if (update_lsn1 != InvalidXLogRecPtr) elog(DEBUG3, "ptrack: update_lsn1 %X/%X of blckno %u of file %s", (uint32) (update_lsn1 >> 32), (uint32) update_lsn1, ctx->bid.blocknum, ctx->relpath); +#endif /* Only probe the second slot if the first one is marked */ if (update_lsn1 >= ctx->lsn) @@ -550,10 +653,12 @@ ptrack_get_pagemapset(PG_FUNCTION_ARGS) slot2 = (size_t)(((hash << 32) | (hash >> 32)) % PtrackContentNblocks); update_lsn2 = pg_atomic_read_u64(&ptrack_map->entries[slot2]); +#if USE_ASSERT_CHECKING if (update_lsn2 != InvalidXLogRecPtr) elog(DEBUG3, "ptrack: update_lsn2 %X/%X of blckno %u of file %s", (uint32) (update_lsn1 >> 32), (uint32) update_lsn2, ctx->bid.blocknum, ctx->relpath); +#endif /* Block has been changed since specified LSN. Mark it in the bitmap */ if (update_lsn2 >= ctx->lsn) diff --git a/ptrack.control b/ptrack.control index ec0af9d..7e3a2b7 100644 --- a/ptrack.control +++ b/ptrack.control @@ -1,5 +1,5 @@ # ptrack extension comment = 'block-level incremental backup engine' -default_version = '2.2' +default_version = '2.4' module_pathname = '$libdir/ptrack' relocatable = true diff --git a/ptrack.h b/ptrack.h index d205115..abeffb3 100644 --- a/ptrack.h +++ b/ptrack.h @@ -4,6 +4,7 @@ * header for ptrack map for tracking updates of relation's pages * * + * Copyright (c) 2019-2022, Postgres Professional * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * @@ -17,14 +18,38 @@ #include "access/xlogdefs.h" #include "storage/block.h" #include "storage/buf.h" +#if PG_VERSION_NUM >= 160000 +#include "storage/relfilelocator.h" +#else #include "storage/relfilenode.h" +#endif #include "storage/smgr.h" #include "utils/relcache.h" /* Ptrack version as a string */ -#define PTRACK_VERSION "2.2" +#define PTRACK_VERSION "2.4" /* Ptrack version as a number */ -#define PTRACK_VERSION_NUM 220 +#define PTRACK_VERSION_NUM 240 +/* Last ptrack version that changed map file format */ +#define PTRACK_MAP_FILE_VERSION_NUM 220 + +#if PG_VERSION_NUM >= 160000 +#define RelFileNode RelFileLocator +#define RelFileNodeBackend RelFileLocatorBackend +#define nodeDb(node) (node).dbOid +#define nodeSpc(node) (node).spcOid +#define nodeRel(node) (node).relNumber +#define nodeOf(ndbck) (ndbck).locator +#else +#define nodeDb(node) (node).dbNode +#define nodeSpc(node) (node).spcNode +#define nodeRel(node) (node).relNode +#define nodeOf(ndbck) (ndbck).node +#endif + +#if PG_VERSION_NUM >= 170000 +#define InvalidBackendId INVALID_PROC_NUMBER +#endif /* * Structure identifying block on the disk. @@ -57,6 +82,7 @@ typedef struct PtrackFileList_i ForkNumber forknum; int segno; char *path; + } PtrackFileList_i; #endif /* PTRACK_H */ diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 90654cc..0000000 --- a/run_tests.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2019-2020, Postgres Professional -# - -PG_SRC=$PWD/postgres -status=0 - -# curl "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" -o postgresql.tar.bz2 -# echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c - - -# mkdir $PG_SRC - -# tar \ -# --extract \ -# --file postgresql.tar.bz2 \ -# --directory $PG_SRC \ -# --strip-components 1 - -# Clone Postgres -echo "############### Getting Postgres sources" -git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 - -# Clone pg_probackup -echo "############### Getting pg_probackup sources" -git clone https://github.com/postgrespro/pg_probackup.git --depth=1 -b master -# git clone https://github.com/ololobus/pg_probackup.git --depth=1 -b ptrack-tests - -# Compile and install Postgres -cd postgres # Go to postgres dir - -echo "############### Applying ptrack patch" -git apply -v -3 ../patches/$PG_BRANCH-ptrack-core.diff - -if [ "$MODE" = "paranoia" ]; then - echo "############### Paranoia mode: applying turn-off-hint-bits.diff" - git apply -v -3 ../patches/turn-off-hint-bits.diff -fi - -echo "############### Compiling Postgres" -if [ "$TEST_CASE" = "tap" ] && [ "$MODE" = "legacy" ]; then - ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests -else - ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests -fi -make -s -j$(nproc) install -make -s -j$(nproc) -C contrib/ install - -# Override default Postgres instance -export PATH=$PGHOME/bin:$PATH -export LD_LIBRARY_PATH=$PGHOME/lib -export PG_CONFIG=$(which pg_config) - -# Show pg_config path (just in case) -echo "############### pg_config path" -which pg_config - -# Show pg_config just in case -echo "############### pg_config" -pg_config - -# Get amcheck if missing -if [ ! -d "contrib/amcheck" ]; then - echo "############### Getting missing amcheck" - git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck - make USE_PGXS=1 -C contrib/amcheck install -fi - -# Get back to testdir -cd .. - -# Build and install ptrack extension -echo "############### Compiling and installing ptrack extension" - -# XXX: Hackish way to make possible to run tap tests -mkdir $PG_SRC/contrib/ptrack -cp * $PG_SRC/contrib/ptrack/ -cp -R t $PG_SRC/contrib/ptrack/ - -make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" -C $PG_SRC/contrib/ptrack/ install - -if [ "$TEST_CASE" = "tap" ]; then - - # Run tap tests - echo "############### Running tap tests" - if [ "$MODE" = "legacy" ]; then - # There is a known issue with attaching shared memory segment using the same - # address each time, when EXEC_BACKEND mechanism is turned on. It happens due - # to the ASLR address space randomization, so we are trying to attach a segment - # to the already occupied location. That way we simply turning off ASLR here. - # - # Postgres comment: https://github.com/postgres/postgres/blob/5cbfce562f7cd2aab0cdc4694ce298ec3567930e/src/backend/postmaster/postmaster.c#L4929 - setarch x86_64 --addr-no-randomize make -C postgres/contrib/ptrack check || status=$? - else - make -C postgres/contrib/ptrack check || status=$? - fi - -else - - # Build and install pg_probackup - echo "############### Compiling and installing pg_probackup" - cd pg_probackup # Go to pg_probackup dir - make USE_PGXS=1 top_srcdir=$PG_SRC install - - # Setup python environment - echo "############### Setting up python env" - virtualenv pyenv - source pyenv/bin/activate - pip install testgres==1.8.2 - - echo "############### Testing" - if [ "$MODE" = "basic" ]; then - export PG_PROBACKUP_TEST_BASIC=ON - elif [ "$MODE" = "paranoia" ]; then - export PG_PROBACKUP_PARANOIA=ON - fi - - if [ "$TEST_CASE" = "all" ]; then - # Run all pg_probackup ptrack tests - python -m unittest -v tests.ptrack || status=$? - else - for i in `seq $TEST_REPEATS`; do - python -m unittest -v tests.ptrack.PtrackTest.$TEST_CASE || status=$? - done - fi - - # Exit virtualenv - deactivate - - # Get back to testdir - cd .. - -fi - -# Generate *.gcov files -gcov $PG_SRC/contrib/ptrack/*.c $PG_SRC/contrib/ptrack/*.h - -# Send coverage stats to Codecov -bash <(curl -s https://codecov.io/bash) - -# Something went wrong, exit with code 1 -if [ $status -ne 0 ]; then exit 1; fi diff --git a/t/001_basic.pl b/t/001_basic.pl index bac81f2..bdb1eca 100644 --- a/t/001_basic.pl +++ b/t/001_basic.pl @@ -6,19 +6,52 @@ use strict; use warnings; -use PostgresNode; -use TestLib; use Test::More; -plan tests => 24; +my $pg_15_modules; + +BEGIN +{ + $pg_15_modules = eval + { + require PostgreSQL::Test::Cluster; + require PostgreSQL::Test::Utils; + return 1; + }; + + unless (defined $pg_15_modules) + { + $pg_15_modules = 0; + + require PostgresNode; + require TestLib; + } +} + +plan tests => 23; + +note('PostgreSQL 15 modules are used: ' . ($pg_15_modules ? 'yes' : 'no')); my $node; my $res; my $res_stdout; my $res_stderr; -# Initialize node -$node = get_new_node('node'); +# Create node. +# Older versions of PostgreSQL modules use get_new_node function. +# Newer use standard perl object constructor syntax. +eval +{ + if ($pg_15_modules) + { + $node = PostgreSQL::Test::Cluster->new("node"); + } + else + { + $node = PostgresNode::get_new_node("node"); + } +}; + $node->init; $node->start; @@ -149,7 +182,6 @@ # Check that we have lost everything ok(! -f $node->data_dir . "/global/ptrack.map", "ptrack.map should be cleaned up"); ok(! -f $node->data_dir . "/global/ptrack.map.tmp", "ptrack.map.tmp should be cleaned up"); -ok(! -f $node->data_dir . "/global/ptrack.map.mmap", "ptrack.map.mmap should be cleaned up"); ($res, $res_stdout, $res_stderr) = $node->psql("postgres", "SELECT ptrack_get_pagemapset('0/0')"); is($res, 3, 'errors out if ptrack is disabled');