From 6db4520f3b2825ec71166a9ffa6cbe0c95ffefef Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Sat, 26 Oct 2024 02:16:34 +0300 Subject: [PATCH 01/10] Fix test test_child_pids for PostgreSql 15 and higher (#146) --- tests/test_simple.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index ba23c3ed..43394718 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -957,6 +957,9 @@ def test_child_pids(self): if pg_version_ge('10'): master_processes.append(ProcessType.LogicalReplicationLauncher) + if pg_version_ge('14'): + master_processes.remove(ProcessType.StatsCollector) + repl_processes = [ ProcessType.Startup, ProcessType.WalReceiver, From 8a5c252ebc5dadd871bb92eec10959a8733167b4 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Wed, 23 Oct 2024 14:57:06 +0300 Subject: [PATCH 02/10] Update configuration for Travis CI. --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index c06cab3d..6f63a67b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,9 @@ notifications: on_failure: always env: + - PYTHON_VERSION=3 PG_VERSION=17 + - PYTHON_VERSION=3 PG_VERSION=16 + - PYTHON_VERSION=3 PG_VERSION=15 - PYTHON_VERSION=3 PG_VERSION=14 - PYTHON_VERSION=3 PG_VERSION=13 - PYTHON_VERSION=3 PG_VERSION=12 @@ -32,3 +35,8 @@ env: # - PYTHON_VERSION=2 PG_VERSION=9.6 # - PYTHON_VERSION=2 PG_VERSION=9.5 # - PYTHON_VERSION=2 PG_VERSION=9.4 + +matrix: + allow_failures: + - env: PYTHON_VERSION=3 PG_VERSION=11 + - env: PYTHON_VERSION=3 PG_VERSION=10 From 569923a004165aa40530444b2154ad98e3f57cc7 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 28 Oct 2024 17:10:17 +0700 Subject: [PATCH 03/10] Configure the unix socket directory --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index b9bf9896..cb1b8385 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1751,7 +1751,8 @@ def make_simple( 'log_connections': 'on', 'log_disconnections': 'on', 'restart_after_crash': 'off', - 'autovacuum': 'off'} + 'autovacuum': 'off', + 'unix_socket_directories': '/tmp'} # Allow replication in pg_hba.conf if set_replication: From c0dca6bf8c9f8fa41ad076542ce9f4a0e68daa2d Mon Sep 17 00:00:00 2001 From: dura0ok Date: Mon, 18 Nov 2024 16:32:33 +0700 Subject: [PATCH 04/10] Add PG_PROBACKUP_WAL_TREE_ENABLED env param (#143) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 4 ++++ testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 1 + 2 files changed, 5 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index e656b66d..899a8092 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -54,6 +54,7 @@ def __init__(self, test_class: unittest.TestCase, self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote + self.wal_tree_enabled = init_params.wal_tree_enabled self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None @@ -185,6 +186,9 @@ def add_instance(self, instance, node, old_binary=False, options=None, expect_er '--remote-proto=ssh', '--remote-host=localhost'] + if self.wal_tree_enabled: + options = options + ['--wal-tree'] + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) def set_config(self, instance, old_binary=False, options=None, expect_error=False): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 2d19e980..b7174a7c 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -170,6 +170,7 @@ def __init__(self): self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) From a5cfdb65f56882a11cc38ddd408420467beac6e7 Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 18 Nov 2024 11:57:51 +0100 Subject: [PATCH 05/10] Add bin_dir to make_simple --- testgres/node.py | 10 ++++++---- tests/test_simple.py | 19 ++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index cb1b8385..c8c8c087 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1707,12 +1707,13 @@ def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations def make_empty( self, base_dir=None, - port=None): + port=None, + bin_dir=None): real_base_dir = os.path.join(self.test_path, base_dir) self.os_ops.rmdirs(real_base_dir, ignore_errors=True) self.os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir, port=port) + node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -1726,10 +1727,11 @@ def make_simple( ptrack_enable=False, initdb_params=[], pg_options={}, - checksum=True): + checksum=True, + bin_dir=None): if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port) + node = self.make_empty(base_dir, port, bin_dir=bin_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) diff --git a/tests/test_simple.py b/tests/test_simple.py index 43394718..8f85a23b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,7 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException + TestgresException, NodeApp from testgres import \ TestgresConfig, \ @@ -1044,6 +1044,23 @@ def test_the_same_port(self): node2.port = node.port node2.init().start() + def test_make_simple_with_bin_dir(self): + with get_new_node() as node: + node.init().start() + bin_dir = node.bin_dir + + app = NodeApp() + correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + + try: + wrong_bin_dir = app.make_empty(base_dir=node.base_dir, bin_dir="wrong/path") + wrong_bin_dir.slow_start() + raise RuntimeError("Error was expected.") # We should not reach this + except FileNotFoundError: + pass # Expected error + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 1cd6cd72ae761ec0b9f34ef8477c8b925de36965 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:22:27 +0100 Subject: [PATCH 06/10] Fix style in test_simple.py (#150) --- tests/test_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8f85a23b..62de1df5 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1044,7 +1044,7 @@ def test_the_same_port(self): node2.port = node.port node2.init().start() - def test_make_simple_with_bin_dir(self): + def test_simple_with_bin_dir(self): with get_new_node() as node: node.init().start() bin_dir = node.bin_dir @@ -1059,7 +1059,7 @@ def test_make_simple_with_bin_dir(self): wrong_bin_dir.slow_start() raise RuntimeError("Error was expected.") # We should not reach this except FileNotFoundError: - pass # Expected error + pass # Expected error if __name__ == '__main__': From 5ae340c9b95430ac9065d58bb1d09882fddf7c30 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Wed, 20 Nov 2024 19:55:47 +0800 Subject: [PATCH 07/10] Fix typos (#152) Found via `codespell -L splitted` --- README.md | 4 ++-- testgres/node.py | 4 ++-- testgres/plugins/pg_probackup2/pg_probackup2/gdb.py | 2 +- tests/test_simple.py | 8 ++++---- tests/test_simple_remote.py | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a2a0ec7e..f0071a90 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,12 @@ with testgres.get_new_node() as node: # ... node stops and its files are about to be removed ``` -There are four API methods for runnig queries: +There are four API methods for running queries: | Command | Description | |----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| | `node.psql(query, ...)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | +| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occurs during the execution, an exception will be thrown. | | `node.execute(query, ...)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | | `node.connect(dbname, ...)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | diff --git a/testgres/node.py b/testgres/node.py index c8c8c087..4ae30908 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1143,7 +1143,7 @@ def restore(self, filename, dbname=None, username=None): filename ] # yapf: disable - # try pg_restore if dump is binary formate, and psql if not + # try pg_restore if dump is binary format, and psql if not try: execute_utility(_params, self.utils_log_name) except ExecUtilException: @@ -1286,7 +1286,7 @@ def set_synchronous_standbys(self, standbys): Args: standbys: either :class:`.First` or :class:`.Any` object specifying - sychronization parameters or just a plain list of + synchronization parameters or just a plain list of :class:`.PostgresNode`s replicas which would be equivalent to passing ``First(1, )``. For PostgreSQL 9.5 and below it is only possible to specify a plain list of standbys as diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py index 0b61da65..ceb1f6a9 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -37,7 +37,7 @@ def __init__(self, cmd, env, attach=False): " to run GDB tests") raise GdbException("No gdb usage possible.") - # Check gdb presense + # Check gdb presence try: gdb_version, _ = subprocess.Popen( ['gdb', '--version'], diff --git a/tests/test_simple.py b/tests/test_simple.py index 62de1df5..41203a65 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -501,7 +501,7 @@ def test_logical_replication(self): sub.disable() node1.safe_psql('insert into test values (3, 3)') - # enable and ensure that data successfully transfered + # enable and ensure that data successfully transferred sub.enable() sub.catchup() res = node2.execute('select * from test') @@ -509,7 +509,7 @@ def test_logical_replication(self): # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() + # to explicitly perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) @@ -526,7 +526,7 @@ def test_logical_replication(self): pub.drop() # create new publication and subscription for specific table - # (ommitting copying data as it's already done) + # (omitting copying data as it's already done) pub = node1.publish('newpub', tables=['test']) sub = node2.subscribe(pub, 'newsub', copy_data=False) @@ -535,7 +535,7 @@ def test_logical_replication(self): res = node2.execute('select * from test') self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - # explicitely add table + # explicitly add table with self.assertRaises(ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index d51820ba..79bdb74c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -480,7 +480,7 @@ def test_logical_replication(self): sub.disable() node1.safe_psql('insert into test values (3, 3)') - # enable and ensure that data successfully transfered + # enable and ensure that data successfully transferred sub.enable() sub.catchup() res = node2.execute('select * from test') @@ -488,7 +488,7 @@ def test_logical_replication(self): # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() + # to explicitly perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) @@ -505,7 +505,7 @@ def test_logical_replication(self): pub.drop() # create new publication and subscription for specific table - # (ommitting copying data as it's already done) + # (omitting copying data as it's already done) pub = node1.publish('newpub', tables=['test']) sub = node2.subscribe(pub, 'newsub', copy_data=False) @@ -514,7 +514,7 @@ def test_logical_replication(self): res = node2.execute('select * from test') self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - # explicitely add table + # explicitly add table with self.assertRaises(ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) From 644eea8cd6c2e85536a1d18a5a8b6e5d0b949bce Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:56:33 +0300 Subject: [PATCH 08/10] Refinement of the run function in app.py (#148) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 899a8092..57492814 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -74,6 +74,8 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, command = [command[0], *use_backup_dir.pb_args, *command[1:]] elif use_backup_dir: command = [command[0], *self.backup_dir.pb_args, *command[1:]] + else: + command = [command[0], *self.backup_dir.pb_args[2:], *command[1:]] if not self.probackup_old_path and old_binary: logging.error('PGPROBACKUPBIN_OLD is not set') From b5855e417cdba94766158b0d1fe23c90b717a967 Mon Sep 17 00:00:00 2001 From: MetalDream666 <61190185+MetalDream666@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:30:39 +0300 Subject: [PATCH 09/10] Fix broken pipe from gdb after killing process (#151) --- testgres/plugins/pg_probackup2/pg_probackup2/gdb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py index ceb1f6a9..2424c04d 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -108,6 +108,9 @@ def kill(self): self.proc.stdin.close() self.proc.stdout.close() + def terminate_subprocess(self): + self._execute('kill') + def set_breakpoint(self, location): result = self._execute('break ' + location) From 655a386117c995618720f4eb0ec7cbc990221343 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 29 Nov 2024 19:00:07 +0700 Subject: [PATCH 10/10] Up version --- setup.py | 2 +- testgres/plugins/pg_probackup2/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..a41094d6 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.3', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 2ff5a503..ade2d85d 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.3', + version='0.0.4', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2',