diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/backup.py b/testgres/backup.py index a89e214d..cecb0f7b 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -33,7 +33,8 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=XLogMethod.fetch): + xlog_method=XLogMethod.fetch, + options=None): """ Create a new backup. @@ -43,6 +44,8 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + if not options: + options = [] self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -77,6 +80,7 @@ def __init__(self, "-D", data_dir, "-X", xlog_method.value ] # yapf: disable + _params += options execute_utility(_params, self.log_file) def __enter__(self): diff --git a/testgres/node.py b/testgres/node.py index 0f1dcf98..e5e8fd5f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -726,14 +726,30 @@ def start(self, params=[], wait=True): "start" ] + params # yapf: disable - try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if error and 'does not exist' in error: - raise Exception - except Exception as e: - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) + startup_retries = 5 + while True: + try: + exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: + raise Exception + except Exception as e: + files = self._collect_special_files() + if any(len(file) > 1 and 'Is another postmaster already ' + 'running on port' in file[1].decode() for + file in files): + print("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) + self.port = reserve_port() + options = {} + options['port'] = str(self.port) + self.set_auto_conf(options) + startup_retries -= 1 + time.sleep(5) + continue + + msg = 'Cannot start node' + raise_from(StartNodeException(msg, files), e) + break self._maybe_start_logger() self.is_started = True return self @@ -1355,7 +1371,7 @@ def pgbench(self, username=None, stdout=None, stderr=None, - options=[]): + options=None): """ Spawn a pgbench process. @@ -1369,6 +1385,8 @@ def pgbench(self, Returns: Process created by subprocess.Popen. """ + if options is None: + options = [] # Set default arguments dbname = dbname or default_dbname() @@ -1388,6 +1406,29 @@ def pgbench(self, return proc + def pgbench_with_wait(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Do pgbench command and wait. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + """ + if options is None: + options = [] + + with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: + pgbench.wait() + return + def pgbench_init(self, **kwargs): """ Small wrapper for pgbench_run(). @@ -1598,7 +1639,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node): + def upgrade_from(self, old_node, options=None): """ Upgrade this node from an old node using pg_upgrade. @@ -1611,6 +1652,9 @@ def upgrade_from(self, old_node): if not os.path.exists(self.data_dir): self.init() + if not options: + options = [] + pg_upgrade_binary = self._get_bin_path("pg_upgrade") if not os.path.exists(pg_upgrade_binary): @@ -1625,6 +1669,7 @@ def upgrade_from(self, old_node): "--old-port", str(old_node.port), "--new-port", str(self.port), ] + upgrade_command += options return self.os_ops.exec_command(upgrade_command) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 94dcd997..1a4ca9e7 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -56,6 +56,7 @@ def __init__(self, test_class: unittest.TestCase, self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None + self.execution_time = None def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, skip_log_directory=False, expect_error=False, use_backup_dir=True): @@ -113,16 +114,17 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + start_time = time.time() self.test_class.output = subprocess.check_output( cmdline, stderr=subprocess.STDOUT, env=env ).decode('utf-8', errors='replace') + end_time = time.time() + self.execution_time = end_time - start_time + if command[0] == 'backup' and return_id: - # return backup ID - for line in self.test_class.output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - result = line.split()[2] + result = self.get_backup_id() else: result = self.test_class.output if expect_error is True: @@ -139,6 +141,19 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, else: raise ProbackupException(self.test_class.output, self.test_class.cmd) + def get_backup_id(self): + if init_params.major_version > 2: + pattern = re.compile(r"Backup (.*) completed successfully.") + for line in self.test_class.output.splitlines(): + match = pattern.search(line) + if match: + return match.group(1) + else: + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + return None + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): if options is None: options = [] @@ -149,13 +164,19 @@ def init(self, options=None, old_binary=False, skip_log_directory=False, expect_ use_backup_dir=use_backup_dir ) - def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False): + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False): if options is None: options = [] + + if not datname: + datname = 'postgres' + cmd = [ 'add-instance', '--instance={0}'.format(instance), - '-D', node.data_dir + '-D', node.data_dir, + '--pgport', '%i' % node.port, + '--pgdatabase', datname ] # don`t forget to kill old_binary after remote ssh release diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index e3dd9e4f..73731a6e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,6 +204,12 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + if self.probackup_version.split('.')[0].isdigit(): + self.major_version = int(self.probackup_version.split('.')[0]) + else: + print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) + sys.exit(1) + def test_env(self): return self._test_env.copy() diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 381d7ae2..2ff5a503 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.2', + version='0.0.3', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', diff --git a/tests/test_simple.py b/tests/test_simple.py index 8cb0d94e..ba23c3ed 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1033,6 +1033,14 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + def test_the_same_port(self): + with get_new_node() as node: + node.init().start() + + with get_new_node() as node2: + node2.port = node.port + node2.init().start() + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'):