From 4c70b80b84da5d0c950b645b28aee85593374239 Mon Sep 17 00:00:00 2001 From: egarbuz <165897130+egarbuz@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:02:39 +0300 Subject: [PATCH 01/17] Adding port and database name into params list for add-instance command (#118) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 94dcd997..1ab71109 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -149,13 +149,19 @@ def init(self, options=None, old_binary=False, skip_log_directory=False, expect_ use_backup_dir=use_backup_dir ) - def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False): + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False): if options is None: options = [] + + if not datname: + datname = 'postgres' + cmd = [ 'add-instance', '--instance={0}'.format(instance), - '-D', node.data_dir + '-D', node.data_dir, + '--pgport', '%i' % node.port, + '--pgdatabase', datname ] # don`t forget to kill old_binary after remote ssh release From 3dc640fb7d6df7e5bbdd1970376773f3d9b7bd09 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Thu, 4 Apr 2024 14:11:09 +0300 Subject: [PATCH 02/17] Value pg_probackup major_version added to the class Init --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index e3dd9e4f..b6bf90f4 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,6 +204,8 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + self.major_version = int(self.probackup_version.split('.')[0]) + def test_env(self): return self._test_env.copy() From fba3bd36a6f86c6be42f87f3fb10e873b02dbec3 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Thu, 4 Apr 2024 19:16:02 +0300 Subject: [PATCH 03/17] Fix error 'Is another postmaster already running on port XXX' Sometimes when we abnormally shutdown node its port stays busy. So I added retry attempts to start() function in case we encounter such error. Test for this case was added (test_the_same_port). --- testgres/node.py | 32 ++++++++++++++++++++++++-------- tests/test_simple.py | 8 ++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0f1dcf98..c9b34d7b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -726,14 +726,30 @@ def start(self, params=[], wait=True): "start" ] + params # yapf: disable - try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if error and 'does not exist' in error: - raise Exception - except Exception as e: - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) + startup_retries = 5 + while True: + try: + exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: + raise Exception + except Exception as e: + files = self._collect_special_files() + if any(len(file) > 1 and 'Is another postmaster already ' + 'running on port' in file[1].decode() for + file in files): + print("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) + self.port = reserve_port() + options = {} + options['port'] = str(self.port) + self.set_auto_conf(options) + startup_retries -= 1 + time.sleep(5) + continue + + msg = 'Cannot start node' + raise_from(StartNodeException(msg, files), e) + break self._maybe_start_logger() self.is_started = True return self diff --git a/tests/test_simple.py b/tests/test_simple.py index 8cb0d94e..ba23c3ed 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1033,6 +1033,14 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + def test_the_same_port(self): + with get_new_node() as node: + node.init().start() + + with get_new_node() as node2: + node2.port = node.port + node2.init().start() + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 2a0c37b9e139cd0a2fc40eedc4edff1fa96dbf66 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Mon, 8 Apr 2024 17:57:55 +0300 Subject: [PATCH 04/17] Add check on digit for probackup_version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index b6bf90f4..109a411e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,7 +204,9 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = int(self.probackup_version.split('.')[0]) + self.major_version = 0 + if self.probackup_version.split('.')[0].isdigit: + self.major_version = int(self.probackup_version.split('.')[0]) def test_env(self): return self._test_env.copy() From 5db4ae393e4b4f69b5721b27e9c80fc3dcd5c044 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:01:07 +0300 Subject: [PATCH 05/17] Break when pg_probackup version is incorrect --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 109a411e..318877af 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,9 +204,11 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = 0 if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) + else: + print('Pg_probackup version is not correct!') + sys.exit(1) def test_env(self): return self._test_env.copy() From 3a90ecd41293aa08c2db662ce73ecad0f34b8636 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:36:42 +0300 Subject: [PATCH 06/17] Adding print incorrect version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 318877af..f8c9f3b3 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version is not correct!') + print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) sys.exit(1) def test_env(self): From ea05a4afc54a9ed65769921b9221afec378a6df1 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 09:45:17 +0300 Subject: [PATCH 07/17] Correction of remarks --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f8c9f3b3..9231ac04 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,10 +204,10 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - if self.probackup_version.split('.')[0].isdigit: + if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) + print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) sys.exit(1) def test_env(self): From 5b8a8369cbc0517c4aecacfa35b2955d8e9a4a0d Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 14:23:31 +0300 Subject: [PATCH 08/17] Changed error message --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 9231ac04..73731a6e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) + print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): From eb0476209523b4809a4a7e20e4f2cd9968dd5cb9 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 15:53:49 +0300 Subject: [PATCH 09/17] Add options to pg_basebackup (#121) --- testgres/backup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/backup.py b/testgres/backup.py index a89e214d..cecb0f7b 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -33,7 +33,8 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=XLogMethod.fetch): + xlog_method=XLogMethod.fetch, + options=None): """ Create a new backup. @@ -43,6 +44,8 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + if not options: + options = [] self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -77,6 +80,7 @@ def __init__(self, "-D", data_dir, "-X", xlog_method.value ] # yapf: disable + _params += options execute_utility(_params, self.log_file) def __enter__(self): From 0cf30a7dc0078645052a7fa9afddb5f6999df5d8 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 20:07:32 +0300 Subject: [PATCH 10/17] Fix port bind (#117) * Fix port bind * Up testgres version - 1.10.1 --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 01251e1c..f182768b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,6 +4,8 @@ import tempfile import platform +from ..utils import reserve_port + # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -392,7 +394,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) + self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) try: conn = pglib.connect( host=host, From 8a25cb375138442c149f93051579a04704bbe38a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 21 May 2024 01:57:29 +0300 Subject: [PATCH 11/17] Add pgbench_with_wait function (#122) --- testgres/node.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index c9b34d7b..d1784cb9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1371,7 +1371,7 @@ def pgbench(self, username=None, stdout=None, stderr=None, - options=[]): + options=None): """ Spawn a pgbench process. @@ -1385,6 +1385,8 @@ def pgbench(self, Returns: Process created by subprocess.Popen. """ + if options is None: + options = [] # Set default arguments dbname = dbname or default_dbname() @@ -1404,6 +1406,29 @@ def pgbench(self, return proc + def pgbench_with_wait(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Do pgbench command and wait. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + """ + if options is None: + options = [] + + with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: + pgbench.wait() + return + def pgbench_init(self, **kwargs): """ Small wrapper for pgbench_run(). From e375302a114cd4df3ceed54d6526f250c44c08e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Fri, 24 May 2024 17:36:14 -0300 Subject: [PATCH 12/17] Add options to pg_upgrade (#125) --- testgres/node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index d1784cb9..e5e8fd5f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1639,7 +1639,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node): + def upgrade_from(self, old_node, options=None): """ Upgrade this node from an old node using pg_upgrade. @@ -1652,6 +1652,9 @@ def upgrade_from(self, old_node): if not os.path.exists(self.data_dir): self.init() + if not options: + options = [] + pg_upgrade_binary = self._get_bin_path("pg_upgrade") if not os.path.exists(pg_upgrade_binary): @@ -1666,6 +1669,7 @@ def upgrade_from(self, old_node): "--old-port", str(old_node.port), "--new-port", str(self.port), ] + upgrade_command += options return self.os_ops.exec_command(upgrade_command) From 63db2752501f9ac58a05accc686aec3cd2825c20 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Tue, 28 May 2024 18:07:38 +0300 Subject: [PATCH 13/17] Added command execution time measurement (#123) This will be useful to keep track of possible performance degradation when code changes. --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1ab71109..3d2e0101 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -56,6 +56,7 @@ def __init__(self, test_class: unittest.TestCase, self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None + self.execution_time = None def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, skip_log_directory=False, expect_error=False, use_backup_dir=True): @@ -113,11 +114,15 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + start_time = time.time() self.test_class.output = subprocess.check_output( cmdline, stderr=subprocess.STDOUT, env=env ).decode('utf-8', errors='replace') + end_time = time.time() + self.execution_time = end_time - start_time + if command[0] == 'backup' and return_id: # return backup ID for line in self.test_class.output.splitlines(): From c1cfc26b4b74c36fd6cccda2ee00fdce60dabc92 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 30 May 2024 11:28:33 +0200 Subject: [PATCH 14/17] Add parsing backup_id in run app.py (#126) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 3d2e0101..1a4ca9e7 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -124,10 +124,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, self.execution_time = end_time - start_time if command[0] == 'backup' and return_id: - # return backup ID - for line in self.test_class.output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - result = line.split()[2] + result = self.get_backup_id() else: result = self.test_class.output if expect_error is True: @@ -144,6 +141,19 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, else: raise ProbackupException(self.test_class.output, self.test_class.cmd) + def get_backup_id(self): + if init_params.major_version > 2: + pattern = re.compile(r"Backup (.*) completed successfully.") + for line in self.test_class.output.splitlines(): + match = pattern.search(line) + if match: + return match.group(1) + else: + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + return None + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): if options is None: options = [] From b4268f8bdaaf67e0f7677c0f801210a0085bb594 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:34:35 +0200 Subject: [PATCH 15/17] Revert "Fix port bind (#117)" (#128) This reverts commit 0cf30a7dc0078645052a7fa9afddb5f6999df5d8. --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..b006c8bf 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.0', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f182768b..01251e1c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,8 +4,6 @@ import tempfile import platform -from ..utils import reserve_port - # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -394,7 +392,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) + self.establish_ssh_tunnel(local_port=port, remote_port=5432) try: conn = pglib.connect( host=host, From 0728583c9b6a9df660cfd209220d8031893d136c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:01:31 +0700 Subject: [PATCH 16/17] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', From 529b4df245ff73b9db757854174f29bb208a0c4c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:16:50 +0700 Subject: [PATCH 17/17] Up pg_probackup2 version --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 381d7ae2..2ff5a503 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.2', + version='0.0.3', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2',