draft start for the plugin Testgres.probackup

Author: Burus

setup.py

@@ -6,7 +6,7 @@
 from distutils.core import setup
 
 # Basic dependencies
-install_requires = ["pg8000", "port-for>=0.4", "six>=1.9.0", "psutil"]
+install_requires = ["pg8000", "port-for>=0.4", "six>=1.9.0", "psutil", "tox", "termcolor"]
 
 # Add compatibility enum class
 if sys.version_info < (3, 4):
@@ -21,7 +21,7 @@
 readme = f.read()
 
 setup(
-version='1.8.5',
+version='1.9.0',
 name='testgres',
 packages=['testgres'],
 description='Testing utility for PostgreSQL and its extensions',

testgres/node.py

@@ -772,6 +772,41 @@ def promote(self, dbname=None, username=None):
 
 return self
 
+def slow_start(self, replica=False):
+
+# wait for https://.com/postgrespro/testgres/pull/50
+# self.start()
+# self.poll_query_until(
+# "postgres",
+# "SELECT not pg_is_in_recovery()",
+# suppress={testgres.NodeConnection})
+
+if replica:
+query = 'SELECT pg_is_in_recovery()'
+else:
+query = 'SELECT not pg_is_in_recovery()'
+
+self.start()
+while True:
+try:
+output = self.safe_psql('template1', query).decode("utf-8").rstrip()
+
+if output == 't':
+break
+
+except testgres.QueryException as e:
+if 'database system is starting up' in e.message:
+pass
+elif 'FATAL: the database system is not accepting connections' in e.message:
+pass
+elif replica and 'Hot standby mode is disabled' in e.message:
+raise e
+else:
+raise e
+
+sleep(0.5)
+
+
 def pg_ctl(self, params):
 """
 Invoke pg_ctl with params.

testgres/probackup.py

@@ -12,13 +12,23 @@
 import re
 import json
 
+import errno
+import unittest
+from termcolor import colored
+
+
 from time import sleep
 from sys import exit, argv, version_info
 
 #from testgres.utils import get_pg_edition
 from testgres.utils import is_enterprise, is_nls_enabled
 
-class ProbackupException(Exception):
+
+# move to the settings.py with other global configurations
+TEST_DIR = 'tmp_dirs'
+
+
+class PGProbackupException(Exception):
 def __init__(self, message, cmd):
 self.message = message
 self.cmd = cmd
@@ -27,21 +37,60 @@ def __str__(self):
 return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd)
 
 
+class PGProbackupApp(object):
+IS_ENTERPRISE = False
+
+settings = dict()
+host = 'localhost'
+tests_full_path = TEST_DIR # tmp_test_dirs
+
+
+#def __new__(self, *args, **kwargs):
+# pass
+
+def __init__(
+self, tests_path='.', test_dir=TEST_DIR,
+*args, **kwargs):
+pass
 
 
-class ProbackupTest(object):
+class PGProbackupTestCaseMixin(unittest.TestCase):
+def setUp(self):
+pass
+
+def tearDown(self):
+pass
+
+
+
+class PGProbackupPackageCompTestCaseMixin(PGProbackupTestCaseMixin):
+current_version = None
+previous_version = None
+
+def setUp(self):
+pass
+
+def tearDown(self):
+pass
+
+
+
+
+class ProbackupTest(object):
 # Class attributes
 enterprise = is_enterprise()
 enable_nls = is_nls_enabled()
 
 def __init__(self, *args, **kwargs):
 super(ProbackupTest, self).__init__(*args, **kwargs)
+
+self.verbose = False
 if '-v' in argv or '--verbose' in argv:
 self.verbose = True
-else:
-self.verbose = False
+
 
 self.test_env = os.environ.copy()
+
 envs_list = [
 'LANGUAGE',
 'LC_ALL',
@@ -75,28 +124,32 @@ def __init__(self, *args, **kwargs):
 self.archive_compress = 'ARCHIVE_COMPRESSION' in self.test_env and \
 self.test_env['ARCHIVE_COMPRESSION'] == 'ON'
 
-try:
-testgres.configure_testgres(
-cache_initdb=False,
-cached_initdb_dir=False,
-cache_pg_config=False,
-node_cleanup_full=False)
-except:
-pass
-
-self.helpers_path = os.path.dirname(os.path.realpath(__file__))
-self.dir_path = os.path.abspath(
-os.path.join(self.helpers_path, os.pardir)
-)
-self.tmp_path = os.path.abspath(
-os.path.join(self.dir_path, 'tmp_dirs')
-)
-try:
-os.makedirs(os.path.join(self.dir_path, 'tmp_dirs'))
-except:
-pass
 
+testgres.configure_testgres(
+cache_initdb=False,
+cached_initdb_dir=False,
+cache_pg_config=False,
+node_cleanup_full=False)
+
+
+# Choose path for the tests artifacts
+self.tests_path =os.path.join(os.path.abspath('.'), TEST_DIR)
+try:
+os.makedirs(os.path.join(self.tests_path))
+log.info('Artefacts from the tests are stored in the direcotry: %s'%self.tests_path)
+except OSError as e:
+if e.errno == errno.EEXIST:
+pass
+else:
+# print(
+# colored('Please choose directory for the tests artefacts with 777 chmod. [%s]'%self.tests_path, 'red'))
+raise
+
+
+# XXX dont work with
 self.user = self.get_username()
+
+
 self.probackup_path = None
 if 'PGPROBACKUPBIN' in self.test_env:
 if (
@@ -108,6 +161,7 @@ def __init__(self, *args, **kwargs):
 if self.verbose:
 print('PGPROBACKUPBIN is not an executable file')
 
+# XXX
 print('@@@', self.probackup_path)
 
 if not self.probackup_path:
@@ -162,18 +216,22 @@ def __init__(self, *args, **kwargs):
 self.old_probackup_version = None
 
 try:
+
 self.probackup_version_output = subprocess.check_output(
 [self.probackup_path, "--version"],
 stderr=subprocess.STDOUT,
 ).decode('utf-8')
+
 except subprocess.CalledProcessError as e:
-raise ProbackupException(e.output.decode('utf-8'))
+raise PGProbackupException(e.output.decode('utf-8'))
 
 if self.probackup_old_path:
+
 old_probackup_version_output = subprocess.check_output(
 [self.probackup_old_path, "--version"],
 stderr=subprocess.STDOUT,
 ).decode('utf-8')
+
 self.old_probackup_version = re.search(
 r"\d+\.\d+\.\d+",
 subprocess.check_output(
@@ -224,19 +282,26 @@ def pg_config_version(self):
 # print('PGPROBACKUP_SSH_USER is not set')
 # exit(1)
 
+
+# XXX move to the node.PostgresNode
 def make_empty_node(
 self,
 base_dir=None):
-real_base_dir = os.path.join(self.tmp_path, base_dir)
+
+real_base_dir = os.path.join(self.tests_path, base_dir)
 shutil.rmtree(real_base_dir, ignore_errors=True)
 os.makedirs(real_base_dir)
 
 node = testgres.get_new_node('test', base_dir=real_base_dir)
+
 # bound method slow_start() to 'node' class instance
-node.slow_start = slow_start.__get__(node)
+#node.slow_start = slow_start.__get__(node)
+
 node.should_rm_dirs = True
 return node
 
+
+# XXX preconfigured Node for the pg_probackup app
 def make_simple_node(
 self,
 base_dir=None,
@@ -254,6 +319,8 @@ def make_simple_node(
 node.major_version_str = str(f.read().rstrip())
 node.major_version = float(node.major_version_str)
 
+# XXX move it to the ProbackupTestCaseMixin class setUp
+
 # Sane default parameters
 options = {}
 options['max_connections'] = 100
@@ -405,7 +472,7 @@ def get_md5_per_page_for_fork(self, file, size_in_pages):
 
 size = size_in_pages
 for segment_number in range(nsegments):
-if size - 131072 > 0:
+if size - 131072 > 0: # XXX make global static config PARAM
 pages_per_segment[segment_number] = 131072
 else:
 pages_per_segment[segment_number] = size
@@ -427,7 +494,7 @@ def get_md5_per_page_for_fork(self, file, size_in_pages):
 for page in range(start_page, end_page):
 md5_per_page[page] = hashlib.md5(
 os.read(file_desc, 8192)).hexdigest()
-offset += 8192
+offset += 8192 # XXX make global static config PARAM
 os.lseek(file_desc, offset, 0)
 os.close(file_desc)
 
@@ -683,7 +750,7 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur
 else:
 return self.output
 except subprocess.CalledProcessError as e:
-raise ProbackupException(e.output.decode('utf-8'), self.cmd)
+raise PGProbackupException(e.output.decode('utf-8'), self.cmd)
 
 def run_binary(self, command, asynchronous=False, env=None):
 
@@ -709,7 +776,7 @@ def run_binary(self, command, asynchronous=False, env=None):
 ).decode('utf-8')
 return self.output
 except subprocess.CalledProcessError as e:
-raise ProbackupException(e.output.decode('utf-8'), command)
+raise PGProbackupException(e.output.decode('utf-8'), command)
 
 def init_pb(self, backup_dir, options=[], old_binary=False):
 
@@ -1455,7 +1522,7 @@ def del_test_dir(self, module_name, fname):
 
 shutil.rmtree(
 os.path.join(
-self.tmp_path,
+self.tests_path,
 module_name,
 fname
 ),

tox.ini

@@ -0,0 +1,9 @@
+[tox]
+envlist = py27,py36
+
+[testenv]
+# install pytest in the virtualenv where commands will be executed
+deps = pytest
+commands =
+# NOTE: you can run any command line tool here - not just tests
+make