Fixes for ptrack tests (test_ptrack_vacuum, test_ptrack_vacuum_bits_frozen, test_ptrack_vacuum_bits_visibility):

this is workaround for spgist metadata update bug (PGPRO-5707)

Author: kulaginm

tests/helpers/ptrack_helpers.py

@@ -1714,8 +1714,30 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None):
 
 return directory_dict
 
-def compare_pgdata(self, original_pgdata, restored_pgdata):
-""" return dict with directory content. DO IT BEFORE RECOVERY"""
+def get_known_bugs_comparision_exclusion_dict(self, node):
+""" get dict of known datafiles difference, that can be used in compare_pgdata() """
+comparision_exclusion_dict = dict()
+
+# bug in spgist metapage update (PGPRO-5707)
+spgist_filelist = node.safe_psql(
+"postgres",
+"SELECT pg_catalog.pg_relation_filepath(pg_class.oid) "
+"FROM pg_am, pg_class "
+"WHERE pg_am.amname = 'spgist' "
+"AND pg_class.relam = pg_am.oid"
+).decode('utf-8').rstrip().splitlines()
+for filename in spgist_filelist:
+comparision_exclusion_dict[filename] = set([0])
+
+return comparision_exclusion_dict
+
+
+def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict()):
+"""
+return dict with directory content. DO IT BEFORE RECOVERY
+exclusion_dict is used for exclude files (and it block_no) from comparision
+it is a dict with relative filenames as keys and set of block numbers as values
+"""
 fail = False
 error_message = 'Restored PGDATA is not equal to original!\n'
 
@@ -1777,16 +1799,17 @@ def compare_pgdata(self, original_pgdata, restored_pgdata):
 original_pgdata['files'][file]['md5'] !=
 restored_pgdata['files'][file]['md5']
 ):
-fail = True
-error_message += (
-'\nFile Checksumm mismatch.\n'
-'File_old: {0}\nChecksumm_old: {1}\n'
-'File_new: {2}\nChecksumm_new: {3}\n').format(
-os.path.join(original_pgdata['pgdata'], file),
-original_pgdata['files'][file]['md5'],
-os.path.join(restored_pgdata['pgdata'], file),
-restored_pgdata['files'][file]['md5']
-)
+if file not in exclusion_dict:
+fail = True
+error_message += (
+'\nFile Checksum mismatch.\n'
+'File_old: {0}\nChecksum_old: {1}\n'
+'File_new: {2}\nChecksum_new: {3}\n').format(
+os.path.join(original_pgdata['pgdata'], file),
+original_pgdata['files'][file]['md5'],
+os.path.join(restored_pgdata['pgdata'], file),
+restored_pgdata['files'][file]['md5']
+)
 
 if original_pgdata['files'][file]['is_datafile']:
 for page in original_pgdata['files'][file]['md5_per_page']:
@@ -1802,13 +1825,16 @@ def compare_pgdata(self, original_pgdata, restored_pgdata):
 )
 continue
 
-if original_pgdata['files'][file][
-'md5_per_page'][page] != restored_pgdata[
-'files'][file]['md5_per_page'][page]:
+if not (file in exclusion_dict and page in exclusion_dict[file]):
+if (
+original_pgdata['files'][file]['md5_per_page'][page] !=
+restored_pgdata['files'][file]['md5_per_page'][page]
+):
+fail = True
 error_message += (
-'\n Page checksumm mismatch: {0}\n '
-' PAGE Checksumm_old: {1}\n '
-' PAGE Checksumm_new: {2}\n '
+'\n Page checksum mismatch: {0}\n '
+' PAGE Checksum_old: {1}\n '
+' PAGE Checksum_new: {2}\n '
 ' File: {3}\n'
 ).format(
 page,

tests/ptrack.py

@@ -3210,6 +3210,8 @@ def test_ptrack_vacuum(self):
 idx_ptrack[i]['type'],
 idx_ptrack[i]['column']))
 
+comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node)
+
 node.safe_psql('postgres', 'vacuum t_heap')
 node.safe_psql('postgres', 'checkpoint')
 
@@ -3253,7 +3255,7 @@ def test_ptrack_vacuum(self):
 self.restore_node(backup_dir, 'node', node)
 
 pgdata_restored = self.pgdata_content(node.data_dir)
-self.compare_pgdata(pgdata, pgdata_restored)
+self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion)
 
 # Clean after yourself
 self.del_test_dir(module_name, self.fname)
@@ -3403,6 +3405,7 @@ def test_ptrack_vacuum_bits_frozen(self):
 idx_ptrack[i]['type'],
 idx_ptrack[i]['column']))
 
+comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node)
 node.safe_psql('postgres', 'checkpoint')
 
 self.backup_node(
@@ -3438,7 +3441,7 @@ def test_ptrack_vacuum_bits_frozen(self):
 self.restore_node(backup_dir, 'node', node)
 
 pgdata_restored = self.pgdata_content(node.data_dir)
-self.compare_pgdata(pgdata, pgdata_restored)
+self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion)
 
 # Clean after yourself
 self.del_test_dir(module_name, self.fname)
@@ -3579,6 +3582,7 @@ def test_ptrack_vacuum_bits_visibility(self):
 i, idx_ptrack[i]['relation'],
 idx_ptrack[i]['type'], idx_ptrack[i]['column']))
 
+comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node)
 node.safe_psql('postgres', 'checkpoint')
 
 self.backup_node(
@@ -3614,7 +3618,7 @@ def test_ptrack_vacuum_bits_visibility(self):
 self.restore_node(backup_dir, 'node', node)
 
 pgdata_restored = self.pgdata_content(node.data_dir)
-self.compare_pgdata(pgdata, pgdata_restored)
+self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion)
 
 # Clean after yourself
 self.del_test_dir(module_name, self.fname)