diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ff03e5de --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +dist +env +venv +*.egg-info diff --git a/.gitignore b/.gitignore index aebd9e97..238181b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,14 @@ *.pyc *.egg *.egg-info/ -dist +.eggs/ +dist/ +build/ +docs/build/ +logs/ env/ venv/ -build/ .coverage coverage.xml diff --git a/.travis.yml b/.travis.yml index 3fd628cb..55b7afa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ -sudo: required +os: linux + +dist: bionic language: python @@ -6,11 +8,10 @@ services: - docker install: - - ./mk_dockerfile.sh - - docker-compose build + - docker build --build-arg PG_VERSION="${PG_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t tests -f Dockerfile--${TEST_PLATFORM}.tmpl . script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests + - docker run $(bash <(curl -s https://codecov.io/env)) -t tests notifications: email: @@ -18,11 +19,19 @@ notifications: on_failure: always env: - - PYTHON_VERSION=2 PG_VERSION=10 - - PYTHON_VERSION=2 PG_VERSION=9.6 - - PYTHON_VERSION=2 PG_VERSION=9.5 - - PYTHON_VERSION=2 PG_VERSION=9.4 - - PYTHON_VERSION=3 PG_VERSION=10 - - PYTHON_VERSION=3 PG_VERSION=9.6 - - PYTHON_VERSION=3 PG_VERSION=9.5 - - PYTHON_VERSION=3 PG_VERSION=9.4 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8.0 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.10 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.11 PG_VERSION=17 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=13 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 + - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=ubuntu_24_04 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=altlinux_10 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=altlinux_11 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl new file mode 100644 index 00000000..d78b05f5 --- /dev/null +++ b/Dockerfile--altlinux_10.tmpl @@ -0,0 +1,118 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM alt:p10 as base1 +ARG PG_VERSION + +RUN apt-get update +RUN apt-get install -y sudo curl ca-certificates +RUN apt-get update +RUN apt-get install -y openssh-server openssh-clients +RUN apt-get install -y time + +# RUN apt-get install -y mc + +RUN apt-get install -y libsqlite3-devel + +EXPOSE 22 + +RUN ssh-keygen -A + +# --------------------------------------------- postgres +FROM base1 as base1_with_dev_tools + +RUN apt-get update + +RUN apt-get install -y git +RUN apt-get install -y gcc +RUN apt-get install -y make + +RUN apt-get install -y meson +RUN apt-get install -y flex +RUN apt-get install -y bison + +RUN apt-get install -y pkg-config +RUN apt-get install -y libssl-devel +RUN apt-get install -y libicu-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y zlib-devel +RUN apt-get install -y liblz4-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y libxml2-devel + +# --------------------------------------------- postgres +FROM base1_with_dev_tools as base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} as base2_with_python-3 +RUN apt-get install -y python3 +RUN apt-get install -y python3-dev +RUN apt-get install -y python3-module-virtualenv +RUN apt-get install -y python3-modules-sqlite3 + +# AltLinux does not have "generic" virtualenv utility. Let's create it. +RUN if [[ -f "/usr/bin/virtualenv" ]] ; then \ + echo AAA; \ + elif [[ -f "/usr/bin/virtualenv3" ]] ; then \ + ln -s /usr/bin/virtualenv3 /usr/bin/virtualenv; \ + else \ + echo "/usr/bin/virtualenv is not created!"; \ + exit 1; \ + fi + +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo \"WHEEL_USERS ALL=(ALL:ALL) NOPASSWD: ALL\"" >> /etc/sudoers + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R test /pg/testgres + +ENV LANG=C.UTF-8 + +USER test + +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# So, we exclude the "remote" tests until this problem has been resolved. +# + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl new file mode 100644 index 00000000..5c88585d --- /dev/null +++ b/Dockerfile--altlinux_11.tmpl @@ -0,0 +1,118 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM alt:p11 as base1 +ARG PG_VERSION + +RUN apt-get update +RUN apt-get install -y sudo curl ca-certificates +RUN apt-get update +RUN apt-get install -y openssh-server openssh-clients +RUN apt-get install -y time + +# RUN apt-get install -y mc + +RUN apt-get install -y libsqlite3-devel + +EXPOSE 22 + +RUN ssh-keygen -A + +# --------------------------------------------- postgres +FROM base1 as base1_with_dev_tools + +RUN apt-get update + +RUN apt-get install -y git +RUN apt-get install -y gcc +RUN apt-get install -y make + +RUN apt-get install -y meson +RUN apt-get install -y flex +RUN apt-get install -y bison + +RUN apt-get install -y pkg-config +RUN apt-get install -y libssl-devel +RUN apt-get install -y libicu-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y zlib-devel +RUN apt-get install -y liblz4-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y libxml2-devel + +# --------------------------------------------- postgres +FROM base1_with_dev_tools as base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} as base2_with_python-3 +RUN apt-get install -y python3 +RUN apt-get install -y python3-dev +RUN apt-get install -y python3-module-virtualenv +RUN apt-get install -y python3-modules-sqlite3 + +# AltLinux does not have "generic" virtualenv utility. Let's create it. +RUN if [[ -f "/usr/bin/virtualenv" ]] ; then \ + echo AAA; \ + elif [[ -f "/usr/bin/virtualenv3" ]] ; then \ + ln -s /usr/bin/virtualenv3 /usr/bin/virtualenv; \ + else \ + echo "/usr/bin/virtualenv is not created!"; \ + exit 1; \ + fi + +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo \"WHEEL_USERS ALL=(ALL:ALL) NOPASSWD: ALL\"" >> /etc/sudoers + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R test /pg/testgres + +ENV LANG=C.UTF-8 + +USER test + +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# So, we exclude the "remote" tests until this problem has been resolved. +# + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/Dockerfile--std-all.tmpl b/Dockerfile--std-all.tmpl new file mode 100644 index 00000000..d19f52a6 --- /dev/null +++ b/Dockerfile--std-all.tmpl @@ -0,0 +1,62 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +#ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +TEST_FILTER=\"\" bash run_tests.sh;" diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl new file mode 100644 index 00000000..67aa30b4 --- /dev/null +++ b/Dockerfile--std.tmpl @@ -0,0 +1,22 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +ENV LANG=C.UTF-8 + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +USER postgres +ENTRYPOINT bash run_tests.sh diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl new file mode 100644 index 00000000..10d8280c --- /dev/null +++ b/Dockerfile--std2-all.tmpl @@ -0,0 +1,96 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers + +# For pyenv +RUN apk add patch +RUN apk add git +RUN apk add xz-dev +RUN apk add zip +RUN apk add zlib-dev +RUN apk add libffi-dev +RUN apk add readline-dev +RUN apk add openssl openssl-dev +RUN apk add sqlite-dev +RUN apk add bzip2-dev + +# --------------------------------------------- base3_with_python-3.8.0 +FROM base2_with_python-3 as base3_with_python-3.8.0 +ENV PYTHON_VERSION=3.8.0 + +# --------------------------------------------- base3_with_python-3.8 +FROM base2_with_python-3 as base3_with_python-3.8 +ENV PYTHON_VERSION=3.8 + +# --------------------------------------------- base3_with_python-3.9 +FROM base2_with_python-3 as base3_with_python-3.9 +ENV PYTHON_VERSION=3.9 + +# --------------------------------------------- base3_with_python-3.10 +FROM base2_with_python-3 as base3_with_python-3.10 +ENV PYTHON_VERSION=3.10 + +# --------------------------------------------- base3_with_python-3.11 +FROM base2_with_python-3 as base3_with_python-3.11 +ENV PYTHON_VERSION=3.11 + +# --------------------------------------------- final +FROM base3_with_python-${PYTHON_VERSION} as final + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash + +RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +export PATH=\"~/.pyenv/bin:$PATH\"; \ +TEST_FILTER=\"\" bash run_tests2.sh;" diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl new file mode 100644 index 00000000..7a559776 --- /dev/null +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -0,0 +1,73 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM ubuntu:24.04 as base1 +ARG PG_VERSION + +RUN apt update +RUN apt install -y sudo curl ca-certificates +RUN apt update +RUN apt install -y openssh-server +RUN apt install -y time +RUN apt install -y netcat-traditional + +RUN apt update +RUN apt install -y postgresql-common + +RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + +RUN install -d /usr/share/postgresql-common/pgdg +RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + +# It does not work +# RUN sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + +RUN apt update +RUN apt install -y postgresql-${PG_VERSION} + +# RUN apt install -y mc + +# [2025-02-26] It adds the user 'postgres' in the group 'sudo' +# [2025-02-27] It is not required. +# RUN adduser postgres sudo + +EXPOSE 22 + +RUN ssh-keygen -A + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apt install -y python3 python3-dev python3-virtualenv libpq-dev +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres /pg + +ENV LANG=C.UTF-8 + +USER postgres + +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + +ENTRYPOINT sh -c " \ +#set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +service ssh enable; \ +sudo service ssh start; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +TEST_FILTER=\"\" bash ./run_tests.sh;" diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl deleted file mode 100644 index 9d417a31..00000000 --- a/Dockerfile.tmpl +++ /dev/null @@ -1,21 +0,0 @@ -FROM postgres:${PG_VERSION}-alpine - -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 py-virtualenv; \ - fi -ENV LANG=C.UTF-8 - -RUN mkdir -p /pg -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -ADD . /pg/testgres -WORKDIR /pg/testgres -RUN chown -R postgres:postgres /pg - -USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh diff --git a/LICENSE b/LICENSE index f57a1836..7e9cc712 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. -Copyright (c) 2016-2017, Postgres Professional +Copyright (c) 2016-2023, Postgres Professional Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index fc1d65b7..00000000 --- a/MANIFEST +++ /dev/null @@ -1,14 +0,0 @@ -# file GENERATED by distutils, do NOT edit -setup.cfg -setup.py -testgres/__init__.py -testgres/api.py -testgres/backup.py -testgres/cache.py -testgres/config.py -testgres/connection.py -testgres/consts.py -testgres/exceptions.py -testgres/logger.py -testgres/node.py -testgres/utils.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..8adcedf3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +include LICENSE +include README.md +include setup.cfg + +recursive-include testgres *.py +recursive-include tests *.py + +global-exclude *.pyc diff --git a/README.md b/README.md index acec819c..defbc8b3 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ -[![Build Status](https://travis-ci.org/postgrespro/testgres.svg?branch=master)](https://travis-ci.org/postgrespro/testgres) +[![Build Status](https://api.travis-ci.com/postgrespro/testgres.svg?branch=master)](https://travis-ci.com/github/postgrespro/testgres) [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres) [![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) +[Documentation](https://postgrespro.github.io/testgres/) + # testgres -PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported. +PostgreSQL testing utility. Python 3.8+ is supported. ## Installation @@ -37,80 +39,34 @@ python my_tests.py ``` -### Logging - -By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. -If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. - -> Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. - -testgres supports [python logging](https://docs.python.org/3.6/library/logging.html), -which means that you can aggregate logs from several nodes into one file: - -```python -import logging - -# write everything to /tmp/testgres.log -logging.basicConfig(filename='/tmp/testgres.log') - -# create two different nodes with logging -node1 = testgres.get_new_node(use_logging=True).init().start() -node2 = testgres.get_new_node(use_logging=True).init().start() - -# execute a few queries -node1.execute('postgres', 'select 1') -node2.execute('postgres', 'select 2') -``` - - ### Examples Here is an example of what you can do with `testgres`: ```python -with testgres.get_new_node('test') as node: - node.init() # run initdb - node.start() # start PostgreSQL - print(node.execute('postgres', 'select 1')) - node.stop() # stop PostgreSQL -``` +# create a node with random name, port, etc +with testgres.get_new_node() as node: -Let's walk through the code. First, you create a new node using: + # run inidb + node.init() -```python -with testgres.get_new_node('master') as node: -``` + # start PostgreSQL + node.start() -or + # execute a query in a default DB + print(node.execute('select 1')) -```python -with testgres.get_new_node('master', '/path/to/DB') as node: +# ... node stops and its files are about to be removed ``` -where `master` is a node's application name. Name matters if you're testing something like replication. -Function `get_new_node()` only creates directory structure in specified directory (or somewhere in '/tmp' if -we did not specify base directory) for cluster. After that, we have to initialize the PostgreSQL cluster: - -```python -node.init() -``` - -This function runs `initdb` command and adds some basic configuration to `postgresql.conf` and `pg_hba.conf` files. -Function `init()` accepts optional parameter `allows_streaming` which configures cluster for streaming replication (default is `False`). -Now we are ready to start: - -```python -node.start() -``` - -Finally, our temporary cluster is able to process queries. There are four ways to run them: +There are four API methods for running queries: | Command | Description | |----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `node.psql(dbname, query)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(dbname, query)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | -| `node.execute(dbname, query)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | -| `node.connect(dbname, username)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | +| `node.psql(query, ...)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | +| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occurs during the execution, an exception will be thrown. | +| `node.execute(query, ...)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | +| `node.connect(dbname, ...)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | The last one is the most powerful: you can use `begin(isolation_level)`, `commit()` and `rollback()`: ```python @@ -120,12 +76,39 @@ with node.connect() as con: con.rollback() ``` -To stop the server, run: + +### Logging + +By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. +If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. + +> Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. + +`testgres` supports [python logging](https://docs.python.org/3.6/library/logging.html), +which means that you can aggregate logs from several nodes into one file: ```python -node.stop() +import logging + +# write everything to /tmp/testgres.log +logging.basicConfig(filename='/tmp/testgres.log') + +# enable logging, and create two different nodes +testgres.configure_testgres(use_python_logging=True) +node1 = testgres.get_new_node().init().start() +node2 = testgres.get_new_node().init().start() + +# execute a few queries +node1.execute('select 1') +node2.execute('select 2') + +# disable logging +testgres.configure_testgres(use_python_logging=False) ``` +Look at `tests/test_simple.py` file for a complete example of the logging +configuration. + ### Backup & replication @@ -134,7 +117,10 @@ It's quite easy to create a backup and start a new replica: ```python with testgres.get_new_node('master') as master: master.init().start() + + # create a backup with master.backup() as backup: + # create and start a new replica replica = backup.spawn_replica('replica').start() @@ -147,7 +133,7 @@ with testgres.get_new_node('master') as master: ### Benchmarks -`testgres` also can help you to make benchmarks using `pgbench` from postgres installation: +`testgres` is also capable of running benchmarks using `pgbench`: ```python with testgres.get_new_node('master') as master: @@ -160,9 +146,64 @@ with testgres.get_new_node('master') as master: ``` +### Custom configuration + +It's often useful to extend default configuration provided by `testgres`. + +`testgres` has `default_conf()` function that helps control some basic +options. The `append_conf()` function can be used to add custom +lines to configuration lines: + +```python +ext_conf = "shared_preload_libraries = 'postgres_fdw'" + +# initialize a new node +with testgres.get_new_node().init() as master: + + # ... do something ... + + # reset main config file + master.default_conf(fsync=True, + allow_streaming=True) + + # add a new config line + master.append_conf('postgresql.conf', ext_conf) +``` + +Note that `default_conf()` is called by `init()` function; both of them overwrite +the configuration file, which means that they should be called before `append_conf()`. + +### Remote mode +Testgres supports the creation of PostgreSQL nodes on a remote host. This is useful when you want to run distributed tests involving multiple nodes spread across different machines. + +To use this feature, you need to use the RemoteOperations class. This feature is only supported with Linux. +Here is an example of how you might set this up: + +```python +from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node + +# Set up connection params +conn_params = ConnectionParams( + host='your_host', # replace with your host + username='user_name', # replace with your username + ssh_key='path_to_ssh_key' # replace with your SSH key path +) +os_ops = RemoteOperations(conn_params) + +# Add remote testgres config before test +TestgresConfig.set_os_ops(os_ops=os_ops) + +# Proceed with your test +def test_basic_query(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + res = node.execute('SELECT 1') + self.assertEqual(res, [(1,)]) +``` + ## Authors -[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia -[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia -[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia -[Yury Zhuravlev](https://github.com/stalkerg) +[Ildar Musin](https://github.com/zilder) +[Dmitry Ivanov](https://github.com/funbringer) +[Ildus Kurbangaliev](https://github.com/ildus) +[Yury Zhuravlev](https://github.com/stalkerg) diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 471ab779..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,2 +0,0 @@ -tests: - build: . diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..f33f6be0 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = testgres +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..354ce2a2 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,15 @@ +# Building the documentation + +Make sure you have `Sphinx` package installed: + +``` +pip install Sphinx +``` + +Then just run + +``` +make html +``` + +Documentation will be built in `build/html` directory. Other output formats are also available; run `make` without arguments to see the options. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..688a850f --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) + +# -- Project information ----------------------------------------------------- + +project = u'testgres' +copyright = u'2016-2023, Postgres Professional' +author = u'Postgres Professional' + +# The short X.Y version +version = u'' +# The full version, including alpha/beta/rc tags +release = u'1.5' + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'testgresdoc' + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'testgres.tex', u'testgres Documentation', + u'Postgres Professional', 'manual'), +] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, 'testgres', u'testgres Documentation', [author], 1)] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'testgres', u'testgres Documentation', author, 'testgres', + 'One line description of project.', 'Miscellaneous'), +] + +# -- Extension configuration ------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..566d9a50 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,95 @@ + +Testgres documentation +====================== + +Testgres is a PostgreSQL testing framework. + +Installation +============ + +To install testgres, run: + +.. code-block:: bash + + pip install testgres + +We encourage you to use ``virtualenv`` for your testing environment. + +Usage +===== + +Environment +----------- + +Note: by default testgres runs ``initdb``, ``pg_ctl``, ``psql`` provided by ``PATH``. + +There are several ways to specify a custom postgres installation: + +- export ``PG_CONFIG`` environment variable pointing to the ``pg_config`` executable; +- export ``PG_BIN`` environment variable pointing to the directory with executable files. + +Example: + +.. code-block:: bash + + export PG_BIN=$HOME/pg_10/bin + python my_tests.py + +Examples +-------- + +Here is an example of what you can do with ``testgres``: + +.. code-block:: python + + # create a node with random name, port, etc + with testgres.get_new_node() as node: + + # run inidb + node.init() + + # start PostgreSQL + node.start() + + # execute a query in a default DB + print(node.execute('select 1')) + + # ... node stops and its files are about to be removed + +Backup & replication +-------------------- + +It's quite easy to create a backup and start a new replica: + +.. code-block:: python + + with testgres.get_new_node('master') as master: + master.init().start() + + # create a backup + with master.backup() as backup: + + # create and start a new replica + replica = backup.spawn_replica('replica').start() + + # catch up with master node + replica.catchup() + + # execute a dummy query + print(replica.execute('postgres', 'select 1')) + +Modules +======= + +.. toctree:: + :maxdepth: 2 + + testgres + + +.. Indices and tables +.. ================== + +.. * :ref:`genindex` +.. * :ref:`modindex` +.. * :ref:`search` diff --git a/docs/source/testgres.rst b/docs/source/testgres.rst new file mode 100644 index 00000000..c3c43f2a --- /dev/null +++ b/docs/source/testgres.rst @@ -0,0 +1,85 @@ +testgres package +================ + +testgres.api +------------ + +.. automodule:: testgres.api + :members: + :undoc-members: + :show-inheritance: + +testgres.backup +--------------- + +.. automodule:: testgres.backup + :members: + :undoc-members: + :show-inheritance: + +testgres.config +--------------- + +.. automodule:: testgres.config + :members: + :undoc-members: + :show-inheritance: + :member-order: groupwise + +testgres.connection +------------------- + +.. automodule:: testgres.connection + :members: + :undoc-members: + :show-inheritance: + +testgres.enums +-------------- + +.. automodule:: testgres.enums + :members: + :undoc-members: + :show-inheritance: + +testgres.exceptions +------------------- + +.. automodule:: testgres.exceptions + :members: + :undoc-members: + :show-inheritance: + +testgres.node +------------- + +.. autoclass:: testgres.node.PostgresNode + :members: + + .. automethod:: __init__ + +.. autoclass:: testgres.node.ProcessProxy + :members: + +testgres.standby +---------------- + +.. automodule:: testgres.standby + :members: + :undoc-members: + :show-inheritance: + +testgres.pubsub +--------------- + +.. automodule:: testgres.pubsub + +.. autoclass:: testgres.node.Publication + :members: + + .. automethod:: __init__ + +.. autoclass:: testgres.node.Subscription + :members: + + .. automethod:: __init__ diff --git a/hooks/README.md b/hooks/README.md new file mode 100644 index 00000000..607a38ea --- /dev/null +++ b/hooks/README.md @@ -0,0 +1,7 @@ +### What's this? + +This is a set of git hooks to be executed on special events, e.g. before you commit your changes. To install them, just execute the `install.sh` script, and you're good to go! + +### What do they do? + +Currently there's only one hook (`pre-commit`) which formats changed python files with `yapf`. diff --git a/hooks/install.sh b/hooks/install.sh new file mode 100755 index 00000000..d1ea0366 --- /dev/null +++ b/hooks/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash +DIR=$(dirname $0) +ln -s -f ../../hooks/pre-commit "$DIR/../.git/hooks/" diff --git a/hooks/pre-commit b/hooks/pre-commit new file mode 100755 index 00000000..52531d14 --- /dev/null +++ b/hooks/pre-commit @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# capture the changed files that have been staged +changed_files=$(git diff --staged --name-only) + +for file in ${changed_files} +do + if [[ "${file##*.}" == "py" ]]; then + if command -v yapf > /dev/null; then + echo "Run yapf on ${file}" + yapf ${file} -i + git add ${file} + fi + + if command -v flake8 > /dev/null; then + echo "Run flake8 on ${file}" + flake8 ${file} + fi + fi +done + diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh deleted file mode 100755 index d2aa3a8a..00000000 --- a/mk_dockerfile.sh +++ /dev/null @@ -1,2 +0,0 @@ -set -eu -sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile.tmpl > Dockerfile diff --git a/publish_package.sh b/publish_package.sh new file mode 100755 index 00000000..9cc56e94 --- /dev/null +++ b/publish_package.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eux + +# prepare environment +venv_path=.venv +rm -rf "$venv_path" +virtualenv "$venv_path" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +. "$venv_path"/bin/activate + +# install utilities +pip3 install setuptools twine + +# create distribution of the package +python3 setup.py sdist bdist_wheel + +# upload dist +twine upload dist/* + +set +eux + diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..9f5fa375 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = tests testgres/plugins/pg_probackup2/pg_probackup2/tests +addopts = --strict-markers +markers = +#log_file = logs/pytest.log +log_file_level = NOTSET +log_file_format = %(levelname)8s [%(asctime)s] %(message)s +log_file_date_format=%Y-%m-%d %H:%M:%S + diff --git a/run_tests.sh b/run_tests.sh index 93851b3c..65c17dbf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,32 +1,28 @@ #!/usr/bin/env bash -# Copyright (c) 2017, Postgres Professional +# Copyright (c) 2017-2023 Postgres Professional set -eux - -# choose python version -echo python version is $PYTHON_VERSION -VIRTUALENV="virtualenv --python=/usr/bin/python$PYTHON_VERSION" -PIP="pip$PYTHON_VERSION" +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +fi # fail early echo check that pg_config is in PATH command -v pg_config -# prepare environment -VENV_PATH=/tmp/testgres_venv +# prepare python environment +VENV_PATH="/tmp/testgres_venv" rm -rf $VENV_PATH -$VIRTUALENV $VENV_PATH +virtualenv --python="/usr/bin/python${PYTHON_VERSION}" "${VENV_PATH}" export VIRTUAL_ENV_DISABLE_PROMPT=1 -source $VENV_PATH/bin/activate - -# install utilities -$PIP install coverage flake8 +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil # install testgres' dependencies export PYTHONPATH=$(pwd) -$PIP install . +# $PIP install . # test code quality flake8 . @@ -38,24 +34,26 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a tests/test_simple.py +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_BIN) -export PG_BIN=$(dirname $(which pg_config)) -time coverage run -a tests/test_simple.py -unset PG_BIN +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_CONFIG) -export PG_CONFIG=$(which pg_config) -time coverage run -a tests/test_simple.py -unset PG_CONFIG +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # show coverage coverage report +# build documentation +cd docs +make html +cd .. # attempt to fix codecov set +eux diff --git a/run_tests2.sh b/run_tests2.sh new file mode 100755 index 00000000..173b19dc --- /dev/null +++ b/run_tests2.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2025 Postgres Professional + +set -eux + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +pyenv virtualenv --force ${PYTHON_VERSION} cur +pyenv activate cur + +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +fi + +# fail early +echo check that pg_config is in PATH +command -v pg_config + +# prepare python environment +VENV_PATH="/tmp/testgres_venv" +rm -rf $VENV_PATH +python -m venv "${VENV_PATH}" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil + +# install testgres' dependencies +export PYTHONPATH=$(pwd) +# $PIP install . + +# test code quality +flake8 . + + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + + +# run tests (PATH) +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_BIN) +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_CONFIG) +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# show coverage +coverage report + +# build documentation +cd docs +make html +cd .. + +# attempt to fix codecov +set +eux + +# send coverage stats to Codecov +bash <(curl -s https://codecov.io/bash) diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 66e6ac53..0b209181 --- a/setup.py +++ b/setup.py @@ -6,7 +6,13 @@ from distutils.core import setup # Basic dependencies -install_requires = ["pg8000", "port-for", "six"] +install_requires = [ + "pg8000", + "port-for>=0.4", + "six>=1.9.0", + "psutil", + "packaging" +] # Add compatibility enum class if sys.version_info < (3, 4): @@ -16,15 +22,22 @@ if sys.version_info < (3, 3): install_requires.append("ipaddress") +# Get contents of README file +with open('README.md', 'r') as f: + readme = f.read() + setup( + version='1.11.0', name='testgres', - packages=['testgres'], - version='1.5.0', + packages=['testgres', 'testgres.operations', 'testgres.impl'], description='Testing utility for PostgreSQL and its extensions', - license='PostgreSQL', - author='Ildar Musin', - author_email='zildermann@gmail.com', url='https://github.com/postgrespro/testgres', - keywords=['testing', 'postgresql'], + long_description=readme, + long_description_content_type='text/markdown', + license='PostgreSQL', + author='Postgres Professional', + author_email='testgres@postgrespro.ru', + keywords=['test', 'testing', 'postgresql'], + install_requires=install_requires, classifiers=[], - install_requires=install_requires) +) diff --git a/testgres/__init__.py b/testgres/__init__.py index 29b017a9..339ae62e 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -1,15 +1,40 @@ -from .api import get_new_node +from .api import get_new_node, get_remote_node from .backup import NodeBackup -from .config import TestgresConfig, configure_testgres + +from .config import \ + TestgresConfig, \ + configure_testgres, \ + scoped_config, \ + push_config, \ + pop_config from .connection import \ - IsolationLevel, \ NodeConnection, \ + DatabaseError, \ InternalError, \ - ProgrammingError + ProgrammingError, \ + OperationalError + +from .exceptions import \ + TestgresException, \ + ExecUtilException, \ + QueryException, \ + TimeoutException, \ + CatchUpException, \ + StartNodeException, \ + InitNodeException, \ + BackupException, \ + InvalidOperationException + +from .enums import \ + XLogMethod, \ + IsolationLevel, \ + NodeStatus, \ + ProcessType, \ + DumpFormat -from .exceptions import * -from .node import NodeStatus, PostgresNode +from .node import PostgresNode, NodeApp +from .node import PortManager from .utils import \ reserve_port, \ @@ -18,3 +43,28 @@ get_bin_path, \ get_pg_config, \ get_pg_version + +from .standby import \ + First, \ + Any + +from .config import testgres_config + +from .operations.os_ops import OsOperations, ConnectionParams +from .operations.local_ops import LocalOperations +from .operations.remote_ops import RemoteOperations + +__all__ = [ + "get_new_node", + "get_remote_node", + "NodeBackup", "testgres_config", + "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", + "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", + "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", + "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", + "PostgresNode", "NodeApp", + "PortManager", + "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", + "First", "Any", + "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" +] diff --git a/testgres/api.py b/testgres/api.py index 685708e1..6a96ee84 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -12,9 +12,9 @@ ... result = node.safe_psql('postgres', 'select 1') ... print(result.decode('utf-8').strip()) ... node.stop() - +PostgresNode(name='...', port=..., base_dir='...') 1 - +PostgresNode(name='...', port=..., base_dir='...') Or: @@ -27,26 +27,26 @@ ... master.execute('postgres', 'insert into test values (0), (1), (2)') ... replica.catchup() # wait until changes are visible ... print(replica.execute('postgres', 'select count(*) from test')) - +PostgresNode(name='...', port=..., base_dir='...') [(3,)] - -Copyright (c) 2016, Postgres Professional """ - from .node import PostgresNode -def get_new_node(name=None, base_dir=None, use_logging=False): +def get_new_node(name=None, base_dir=None, **kwargs): + """ + Simply a wrapper around :class:`.PostgresNode` constructor. + See :meth:`.PostgresNode.__init__` for details. """ - Create a new node (select port automatically). + # NOTE: leave explicit 'name' and 'base_dir' for compatibility + return PostgresNode(name=name, base_dir=base_dir, **kwargs) - Args: - name: node's application name. - base_dir: path to node's data directory. - use_logging: enable python logging. - Returns: - An instance of PostgresNode. +def get_remote_node(name=None, conn_params=None): """ - - return PostgresNode(name=name, base_dir=base_dir, use_logging=use_logging) + Simply a wrapper around :class:`.PostgresNode` constructor for remote node. + See :meth:`.PostgresNode.__init__` for details. + For remote connection you can add the next parameter: + conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username()) + """ + return get_new_node(name=name, conn_params=conn_params) diff --git a/testgres/backup.py b/testgres/backup.py index a4484309..857c46d4 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -1,30 +1,32 @@ # coding: utf-8 import os -import shutil -import tempfile from six import raise_from +from .enums import XLogMethod + from .consts import \ DATA_DIR, \ + TMP_NODE, \ + TMP_BACKUP, \ PG_CONF_FILE, \ - BACKUP_LOG_FILE, \ - DEFAULT_XLOG_METHOD + BACKUP_LOG_FILE from .exceptions import BackupException +from .operations.os_ops import OsOperations + from .utils import \ - get_bin_path, \ - default_username, \ - execute_utility + get_bin_path2, \ + execute_utility2, \ + clean_on_error class NodeBackup(object): """ Smart object responsible for backups """ - @property def log_file(self): return os.path.join(self.base_dir, BACKUP_LOG_FILE) @@ -33,23 +35,37 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=DEFAULT_XLOG_METHOD): + xlog_method=XLogMethod.fetch, + options=None): """ Create a new backup. Args: - node: PostgresNode we're going to backup. + node: :class:`.PostgresNode` we're going to backup. base_dir: where should we store it? username: database user name. xlog_method: none | fetch | stream (see docs) """ + assert node.os_ops is not None + assert isinstance(node.os_ops, OsOperations) + if not options: + options = [] + self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') + # Check arguments + if not isinstance(xlog_method, XLogMethod): + try: + xlog_method = XLogMethod(xlog_method) + except ValueError: + msg = 'Invalid xlog_method "{}"'.format(xlog_method) + raise BackupException(msg) + # Set default arguments - username = username or default_username() - base_dir = base_dir or tempfile.mkdtemp() + username = username or self.os_ops.get_user() + base_dir = base_dir or self.os_ops.mkdtemp(prefix=TMP_BACKUP) # public self.original_node = node @@ -61,16 +77,16 @@ def __init__(self, data_dir = os.path.join(self.base_dir, DATA_DIR) - # yapf: disable _params = [ - get_bin_path("pg_basebackup"), + get_bin_path2(self.os_ops, "pg_basebackup"), "-p", str(node.port), "-h", node.host, "-U", username, "-D", data_dir, - "-X", xlog_method - ] - execute_utility(_params, self.log_file) + "-X", xlog_method.value + ] # yapf: disable + _params += options + execute_utility2(self.os_ops, _params, self.log_file) def __enter__(self): return self @@ -96,14 +112,14 @@ def _prepare_dir(self, destroy): available = not destroy if available: - dest_base_dir = tempfile.mkdtemp() + dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) data1 = os.path.join(self.base_dir, DATA_DIR) data2 = os.path.join(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir - shutil.copytree(data1, data2) + self.os_ops.copytree(data1, data2) except Exception as e: raise_from(BackupException('Failed to copy files'), e) else: @@ -115,61 +131,79 @@ def _prepare_dir(self, destroy): # Return path to new node return dest_base_dir - def spawn_primary(self, name=None, destroy=True, use_logging=False): + def spawn_primary(self, name=None, destroy=True): """ Create a primary node from a backup. Args: name: primary's application name. destroy: should we convert this backup into a node? - use_logging: enable python logging. Returns: - New instance of PostgresNode. + New instance of :class:`.PostgresNode`. """ # Prepare a data directory for this node base_dir = self._prepare_dir(destroy) # Build a new PostgresNode - from .node import PostgresNode - node = PostgresNode(name=name, - base_dir=base_dir, - use_logging=use_logging) + assert self.original_node is not None - # New nodes should always remove dir tree - node._should_rm_dirs = True + if (hasattr(self.original_node, "clone_with_new_name_and_base_dir")): + node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir) + else: + # For backward compatibility + NodeClass = self.original_node.__class__ + node = NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params) - node.append_conf(PG_CONF_FILE, "\n") - node.append_conf(PG_CONF_FILE, "port = {}".format(node.port)) + assert node is not None + assert type(node) == self.original_node.__class__ # noqa: E721 - return node + with clean_on_error(node) as node: - def spawn_replica(self, name=None, destroy=True, use_logging=False): + # New nodes should always remove dir tree + node._should_rm_dirs = True + + # Set a new port + node.append_conf(filename=PG_CONF_FILE, line='\n') + node.append_conf(filename=PG_CONF_FILE, port=node.port) + + return node + + def spawn_replica(self, name=None, destroy=True, slot=None): """ Create a replica of the original node from a backup. Args: name: replica's application name. + slot: create a replication slot with the specified name. destroy: should we convert this backup into a node? - use_logging: enable python logging. Returns: - New instance of PostgresNode. + New instance of :class:`.PostgresNode`. """ # Build a new PostgresNode - node = self.spawn_primary(name=name, - destroy=destroy, - use_logging=use_logging) - - # Assign it a master and a recovery file (private magic) - node._assign_master(self.original_node) - node._create_recovery_conf(username=self.username) + node = self.spawn_primary(name=name, destroy=destroy) + assert node is not None + + try: + # Assign it a master and a recovery file (private magic) + node._assign_master(self.original_node) + node._create_recovery_conf(username=self.username, slot=slot) + except: # noqa: E722 + # TODO: Pass 'final=True' ? + node.cleanup(release_resources=True) + raise return node def cleanup(self): + """ + Remove all files that belong to this backup. + No-op if it's been converted to a PostgresNode (destroy=True). + """ + if self._available: - shutil.rmtree(self.base_dir, ignore_errors=True) self._available = False + self.os_ops.rmdirs(self.base_dir, ignore_errors=True) diff --git a/testgres/cache.py b/testgres/cache.py index 2fad48f8..499cce91 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -1,61 +1,90 @@ # coding: utf-8 -import atexit import os -import shutil -import tempfile from six import raise_from -from .config import TestgresConfig +from .config import testgres_config + +from .consts import XLOG_CONTROL_FILE + +from .defaults import generate_system_id from .exceptions import \ InitNodeException, \ ExecUtilException from .utils import \ - get_bin_path, \ - execute_utility + get_bin_path2, \ + execute_utility2 + +from .operations.local_ops import LocalOperations +from .operations.os_ops import OsOperations -def cached_initdb(data_dir, initdb_logfile, initdb_params=[]): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True): """ Perform initdb or use cached node files. """ - def call_initdb(initdb_dir): - try: - _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] - execute_utility(_params + initdb_params, initdb_logfile) - except ExecUtilException as e: - raise_from(InitNodeException("Failed to run initdb"), e) + assert os_ops is None or isinstance(os_ops, OsOperations) - def rm_cached_data_dir(cached_data_dir): - shutil.rmtree(cached_data_dir, ignore_errors=True) + if os_ops is None: + os_ops = LocalOperations.get_single_instance() - # Call initdb if we have custom params or shouldn't cache it - if initdb_params or not TestgresConfig.cache_initdb: - call_initdb(data_dir) - else: - # Set default temp dir for cached initdb - if TestgresConfig.cached_initdb_dir is None: + assert isinstance(os_ops, OsOperations) + + def make_utility_path(name): + assert name is not None + assert type(name) == str # noqa: E721 - # Create default temp dir - TestgresConfig.cached_initdb_dir = tempfile.mkdtemp() + if bin_path: + return os.path.join(bin_path, name) - # Schedule cleanup - atexit.register(rm_cached_data_dir, - TestgresConfig.cached_initdb_dir) + return get_bin_path2(os_ops, name) + def call_initdb(initdb_dir, log=logfile): + try: + initdb_path = make_utility_path("initdb") + _params = [initdb_path, "-D", initdb_dir, "-N"] + execute_utility2(os_ops, _params + (params or []), log) + except ExecUtilException as e: + raise_from(InitNodeException("Failed to run initdb"), e) + + if params or not testgres_config.cache_initdb or not cached: + call_initdb(data_dir, logfile) + else: # Fetch cached initdb dir - cached_data_dir = TestgresConfig.cached_initdb_dir + cached_data_dir = testgres_config.cached_initdb_dir # Initialize cached initdb - if not os.listdir(cached_data_dir): + + if not os_ops.path_exists(cached_data_dir) or \ + not os_ops.listdir(cached_data_dir): call_initdb(cached_data_dir) try: # Copy cached initdb to current data dir - shutil.copytree(cached_data_dir, data_dir) + os_ops.copytree(cached_data_dir, data_dir) + + # Assign this node a unique system id if asked to + if testgres_config.cached_initdb_unique: + # XXX: write new unique system id to control file + # Some users might rely upon unique system ids, but + # our initdb caching mechanism breaks this contract. + pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) + system_id = generate_system_id() + cur_pg_control = os_ops.read(pg_control, binary=True) + new_pg_control = system_id + cur_pg_control[len(system_id):] + os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True) + + # XXX: build new WAL segment with our system id + _params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"] + execute_utility2(os_ops, _params, logfile) + + except ExecUtilException as e: + msg = "Failed to reset WAL for system id" + raise_from(InitNodeException(msg), e) + except Exception as e: - raise_from(InitNodeException("Failed to copy files"), e) + raise_from(InitNodeException("Failed to spawn a node"), e) diff --git a/testgres/config.py b/testgres/config.py index 4be623c8..55d52426 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -1,30 +1,217 @@ # coding: utf-8 +import atexit +import copy +import logging +import os +import tempfile -class TestgresConfig: - """ - Global config (override default settings). +from contextlib import contextmanager + +from .consts import TMP_CACHE +from .operations.os_ops import OsOperations +from .operations.local_ops import LocalOperations + +log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig(level=log_level, format=log_format) - Attributes: - cache_initdb: shall we use cached initdb instance? - cache_pg_config: shall we cache pg_config results? - cached_initdb_dir: shall we create a temp dir for cached initdb? - node_cleanup_full: shall we remove EVERYTHING (including logs)? - error_log_lines: N of log lines to be included into exception (0=inf). + +class GlobalConfig(object): + """ + Global configuration object which allows user to override default settings. """ + # NOTE: attributes must not be callable or begin with __. cache_initdb = True + """ shall we use cached initdb instance? """ + + cached_initdb_unique = False + """ shall we give new node a unique system id? """ + cache_pg_config = True - cached_initdb_dir = None - node_cleanup_full = True + """ shall we cache pg_config results? """ + + use_python_logging = False + """ enable python logging subsystem (see logger.py). """ + error_log_lines = 20 + """ N of log lines to be shown in exceptions (0=inf). """ + + node_cleanup_full = True + """ shall we remove EVERYTHING (including logs)? """ + + node_cleanup_on_good_exit = True + """ remove base_dir on nominal __exit__(). """ + + node_cleanup_on_bad_exit = False + """ remove base_dir on __exit__() via exception. """ + + _cached_initdb_dir = None + """ underlying class attribute for cached_initdb_dir property """ + + os_ops = LocalOperations.get_single_instance() + """ OsOperation object that allows work on remote host """ + + @property + def cached_initdb_dir(self): + """ path to a temp directory for cached initdb. """ + return self._cached_initdb_dir + + @cached_initdb_dir.setter + def cached_initdb_dir(self, value): + self._cached_initdb_dir = value + + if value: + cached_initdb_dirs.add(value) + return testgres_config.cached_initdb_dir + + @property + def temp_dir(self): + """ path to temp dir containing nodes with default 'base_dir'. """ + return tempfile.tempdir + + @temp_dir.setter + def temp_dir(self, value): + tempfile.tempdir = value + + def __init__(self, **options): + self.update(options) + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def __setattr__(self, name, value): + if name not in self.keys(): + raise TypeError('Unknown option {}'.format(name)) + + super(GlobalConfig, self).__setattr__(name, value) + + def keys(self): + """ + Return a list of all available settings. + """ + + keys = [] + + for key in dir(GlobalConfig): + if not key.startswith('__') and not callable(self[key]): + keys.append(key) + + return keys + + def items(self): + """ + Return setting-value pairs. + """ + + return ((key, self[key]) for key in self.keys()) + + def update(self, config): + """ + Extract setting-value pairs from 'config' and + assign those values to corresponding settings + of this GlobalConfig object. + """ + + for key, value in config.items(): + self[key] = value + + return self + + def copy(self): + """ + Return a copy of this object. + """ + + return copy.copy(self) + + @staticmethod + def set_os_ops(os_ops: OsOperations): + testgres_config.os_ops = os_ops + testgres_config.cached_initdb_dir = os_ops.mkdtemp(prefix=TMP_CACHE) + + +# cached dirs to be removed +cached_initdb_dirs = set() + +# default config object +testgres_config = GlobalConfig() + +# NOTE: for compatibility +TestgresConfig = testgres_config + +# stack of GlobalConfigs +config_stack = [] + + +@atexit.register +def _rm_cached_initdb_dirs(): + for d in cached_initdb_dirs: + testgres_config.os_ops.rmdirs(d, ignore_errors=True) + + +def push_config(**options): + """ + Permanently set custom GlobalConfig options and + put previous settings on top of the config stack. + """ + + # push current config to stack + config_stack.append(testgres_config.copy()) + + return testgres_config.update(options) + + +def pop_config(): + """ + Set previous GlobalConfig options from stack. + """ + + if len(config_stack) == 0: + raise IndexError('Reached initial config') + + # restore popped config + return testgres_config.update(config_stack.pop()) + + +@contextmanager +def scoped_config(**options): + """ + Temporarily set custom GlobalConfig options for this context. + Previous options are pushed to the config stack. + + Example: + >>> from .api import get_new_node + >>> with scoped_config(cache_initdb=False): + ... # create a new node with fresh initdb + ... with get_new_node().init().start() as node: + ... print(node.execute('select 1')) + [(1,)] + """ + + try: + # set a new config with options + config = push_config(**options) + + # return it + yield config + finally: + # restore previous config + pop_config() def configure_testgres(**options): """ - Configure testgres. - Look at TestgresConfig to check what can be changed. + Adjust current global options. + Look at the GlobalConfig to learn about existing settings. """ - for key, option in options.items(): - setattr(TestgresConfig, key, option) + testgres_config.update(options) + + +# NOTE: assign initial cached dir for initdb +testgres_config.cached_initdb_dir = testgres_config.os_ops.mkdtemp(prefix=TMP_CACHE) diff --git a/testgres/connection.py b/testgres/connection.py index e86ff06d..b8dc49a9 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -1,4 +1,5 @@ # coding: utf-8 +import logging # we support both pg8000 and psycopg2 try: @@ -9,33 +10,31 @@ except ImportError: raise ImportError("You must have psycopg2 or pg8000 modules installed") -from enum import Enum +from .enums import IsolationLevel -from .exceptions import QueryException - -from .utils import \ +from .defaults import \ default_dbname, \ default_username -# export these exceptions +from .exceptions import QueryException + +# export some exceptions +DatabaseError = pglib.DatabaseError InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError - - -class IsolationLevel(Enum): - """ - Transaction isolation level for NodeConnection - """ - - ReadUncommitted, ReadCommitted, RepeatableRead, Serializable = range(4) +OperationalError = pglib.OperationalError class NodeConnection(object): """ Transaction wrapper returned by Node """ - - def __init__(self, node, dbname=None, username=None, password=None): + def __init__(self, + node, + dbname=None, + username=None, + password=None, + autocommit=False): # Set default arguments dbname = dbname or default_dbname() @@ -48,8 +47,10 @@ def __init__(self, node, dbname=None, username=None, password=None): user=username, password=password, host=node.host, - port=node.port) + port=node.port + ) + self._connection.autocommit = autocommit self._cursor = self.connection.cursor() @property @@ -60,6 +61,10 @@ def node(self): def connection(self): return self._connection + @property + def pid(self): + return self.execute("select pg_catalog.pg_backend_pid()")[0][0] + @property def cursor(self): return self._cursor @@ -71,39 +76,21 @@ def __exit__(self, type, value, traceback): self.close() def begin(self, isolation_level=IsolationLevel.ReadCommitted): - # yapf: disable - levels = [ - 'read uncommitted', - 'read committed', - 'repeatable read', - 'serializable' - ] - - # Check if level is an IsolationLevel - if (isinstance(isolation_level, IsolationLevel)): - - # Get index of isolation level - level_idx = isolation_level.value - assert level_idx in range(4) - - # Replace isolation level with its name - isolation_level = levels[level_idx] - - else: + # Check if level isn't an IsolationLevel + if not isinstance(isolation_level, IsolationLevel): # Get name of isolation level level_str = str(isolation_level).lower() # Validate level string - if level_str not in levels: + try: + isolation_level = IsolationLevel(level_str) + except ValueError: error = 'Invalid isolation level "{}"' raise QueryException(error.format(level_str)) - # Replace isolation level with its name - isolation_level = level_str - # Set isolation level cmd = 'SET TRANSACTION ISOLATION LEVEL {}' - self.cursor.execute(cmd.format(isolation_level)) + self.cursor.execute(cmd.format(isolation_level.value)) return self @@ -119,16 +106,14 @@ def rollback(self): def execute(self, query, *args): self.cursor.execute(query, args) - try: - res = self.cursor.fetchall() - # pg8000 might return tuples - if isinstance(res, tuple): - res = [tuple(t) for t in res] - + res = [tuple(t) for t in self.cursor.fetchall()] return res - except Exception: + except ProgrammingError: + return None + except Exception as e: + logging.error("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): diff --git a/testgres/consts.py b/testgres/consts.py index e42a4fa7..89c49ab7 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -4,8 +4,20 @@ DATA_DIR = "data" LOGS_DIR = "logs" +# prefixes for temp dirs +TMP_NODE = 'tgsn_' +TMP_DUMP = 'tgsd_' +TMP_CACHE = 'tgsc_' +TMP_BACKUP = 'tgsb_' + +# path to control file +XLOG_CONTROL_FILE = "global/pg_control" + +# names for config files RECOVERY_CONF_FILE = "recovery.conf" +PG_AUTO_CONF_FILE = "postgresql.auto.conf" PG_CONF_FILE = "postgresql.conf" +PG_PID_FILE = 'postmaster.pid' HBA_CONF_FILE = "pg_hba.conf" # names for log files @@ -13,5 +25,17 @@ UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" -# default argument value -DEFAULT_XLOG_METHOD = "fetch" +# defaults for node settings +MAX_LOGICAL_REPLICATION_WORKERS = 5 +MAX_REPLICATION_SLOTS = 10 +MAX_WORKER_PROCESSES = 10 +WAL_KEEP_SEGMENTS = 20 +WAL_KEEP_SIZE = 320 +MAX_WAL_SENDERS = 10 + +# logical replication settings +LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60 + +PG_CTL__STATUS__OK = 0 +PG_CTL__STATUS__NODE_IS_STOPPED = 3 +PG_CTL__STATUS__BAD_DATADIR = 4 diff --git a/testgres/decorators.py b/testgres/decorators.py new file mode 100644 index 00000000..7f383ae7 --- /dev/null +++ b/testgres/decorators.py @@ -0,0 +1,73 @@ +import six +import functools + + +def positional_args_hack(*special_cases): + """ + Convert positional args described by + 'special_cases' into named args. + + Example: + @positional_args_hack(['abc'], ['def', 'abc']) + def some_api_func(...) + + This is useful for compatibility. + """ + + cases = dict() + + for case in special_cases: + k = len(case) + assert k not in six.iterkeys(cases), 'len must be unique' + cases[k] = case + + def decorator(function): + @functools.wraps(function) + def wrapper(*args, **kwargs): + k = len(args) + + if k in six.iterkeys(cases): + case = cases[k] + + for i in range(0, k): + arg_name = case[i] + arg_val = args[i] + + # transform into named + kwargs[arg_name] = arg_val + + # get rid of them + args = [] + + return function(*args, **kwargs) + + return wrapper + + return decorator + + +def method_decorator(decorator): + """ + Convert a function decorator into a method decorator. + """ + def _dec(func): + def _wrapper(self, *args, **kwargs): + @decorator + def bound_func(*args2, **kwargs2): + return func.__get__(self, type(self))(*args2, **kwargs2) + + # 'bound_func' is a closure and can see 'self' + return bound_func(*args, **kwargs) + + # preserve docs + functools.update_wrapper(_wrapper, func) + + return _wrapper + + # preserve docs + functools.update_wrapper(_dec, decorator) + + # change name for easier debugging + _dec.__name__ = 'method_decorator({})'.format(decorator.__name__) + + return _dec diff --git a/testgres/defaults.py b/testgres/defaults.py new file mode 100644 index 00000000..d77361d7 --- /dev/null +++ b/testgres/defaults.py @@ -0,0 +1,49 @@ +import datetime +import struct +import uuid + +from .config import testgres_config as tconf + + +def default_dbname(): + """ + Return default DB name. + """ + + return 'postgres' + + +def default_username(): + """ + Return default username (current user). + """ + return tconf.os_ops.get_user() + + +def generate_app_name(): + """ + Generate a new application name for node. + """ + + return 'testgres-{}'.format(str(uuid.uuid4())) + + +def generate_system_id(): + """ + Generate a new 64-bit unique system identifier for node. + """ + + date1 = datetime.datetime.utcfromtimestamp(0) + date2 = datetime.datetime.utcnow() + + secs = int((date2 - date1).total_seconds()) + usecs = date2.microsecond + + # see pg_resetwal.c : GuessControlValues() + system_id = 0 + system_id |= (secs << 32) + system_id |= (usecs << 12) + system_id |= (tconf.os_ops.get_pid() & 0xFFF) + + # pack ULL in native byte order + return struct.pack('=Q', system_id) diff --git a/testgres/enums.py b/testgres/enums.py new file mode 100644 index 00000000..d07d8068 --- /dev/null +++ b/testgres/enums.py @@ -0,0 +1,103 @@ +from enum import Enum, IntEnum +from six import iteritems +from psutil import NoSuchProcess + + +class XLogMethod(Enum): + """ + Available WAL methods for :class:`.NodeBackup` + """ + + none = 'none' + fetch = 'fetch' + stream = 'stream' + + +class IsolationLevel(Enum): + """ + Transaction isolation level for :class:`.NodeConnection` + """ + + ReadUncommitted = 'read uncommitted' + ReadCommitted = 'read committed' + RepeatableRead = 'repeatable read' + Serializable = 'serializable' + + +class NodeStatus(IntEnum): + """ + Status of a PostgresNode + """ + + Running, Stopped, Uninitialized = range(3) + + # for Python 3.x + def __bool__(self): + return self == NodeStatus.Running + + # for Python 2.x + __nonzero__ = __bool__ + + +class ProcessType(Enum): + """ + Types of processes + """ + + AutovacuumLauncher = 'autovacuum launcher' + BackgroundWriter = 'background writer' + Checkpointer = 'checkpointer' + LogicalReplicationLauncher = 'logical replication launcher' + Startup = 'startup' + StatsCollector = 'stats collector' + WalReceiver = 'wal receiver' + WalSender = 'wal sender' + WalWriter = 'wal writer' + + # special value + Unknown = 'unknown' + + @staticmethod + def from_process(process): + # legacy names for older releases of PG + alternative_names = { + ProcessType.LogicalReplicationLauncher: [ + 'logical replication worker' + ], + ProcessType.BackgroundWriter: [ + 'writer' + ], + } # yapf: disable + + try: + cmdline = ''.join(process.cmdline()) + except (FileNotFoundError, ProcessLookupError, NoSuchProcess): + return ProcessType.Unknown + + # we deliberately cut special words and spaces + cmdline = cmdline.replace('postgres:', '', 1) \ + .replace('bgworker:', '', 1) \ + .replace(' ', '') + + for ptype in ProcessType: + if cmdline.startswith(ptype.value.replace(' ', '')): + return ptype + + for ptype, names in iteritems(alternative_names): + for name in names: + if cmdline.startswith(name.replace(' ', '')): + return ptype + + # default + return ProcessType.Unknown + + +class DumpFormat(Enum): + """ + Available dump formats + """ + + Plain = 'plain' + Custom = 'custom' + Directory = 'directory' + Tar = 'tar' diff --git a/testgres/exceptions.py b/testgres/exceptions.py index a5578759..20c1a8cf 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -1,38 +1,104 @@ # coding: utf-8 +import six + class TestgresException(Exception): - """ - Base exception - """ + pass + +class PortForException(TestgresException): pass +@six.python_2_unicode_compatible class ExecUtilException(TestgresException): - """ - Stores exit code - """ - - def __init__(self, message, exit_code=0): + def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): super(ExecUtilException, self).__init__(message) + + self.message = message + self.command = command self.exit_code = exit_code + self.out = out + self.error = error + def __str__(self): + msg = [] -class ClusterTestgresException(TestgresException): - pass + if self.message: + msg.append(self.message) + + if self.command: + command_s = ' '.join(self.command) if isinstance(self.command, list) else self.command, + msg.append(u'Command: {}'.format(command_s)) + + if self.exit_code: + msg.append(u'Exit code: {}'.format(self.exit_code)) + + if self.error: + msg.append(u'---- Error:\n{}'.format(self.error)) + + if self.out: + msg.append(u'---- Out:\n{}'.format(self.out)) + + return self.convert_and_join(msg) + + @staticmethod + def convert_and_join(msg_list): + # Convert each byte element in the list to str + str_list = [six.text_type(item, 'utf-8') if isinstance(item, bytes) else six.text_type(item) for item in + msg_list] + + # Join the list into a single string with the specified delimiter + return six.text_type('\n').join(str_list) +@six.python_2_unicode_compatible class QueryException(TestgresException): + def __init__(self, message=None, query=None): + super(QueryException, self).__init__(message) + + self.message = message + self.query = query + + def __str__(self): + msg = [] + + if self.message: + msg.append(self.message) + + if self.query: + msg.append(u'Query: {}'.format(self.query)) + + return six.text_type('\n').join(msg) + + +class TimeoutException(QueryException): pass -class TimeoutException(TestgresException): +class CatchUpException(QueryException): pass +@six.python_2_unicode_compatible class StartNodeException(TestgresException): - pass + def __init__(self, message=None, files=None): + super(StartNodeException, self).__init__(message) + + self.message = message + self.files = files + + def __str__(self): + msg = [] + + if self.message: + msg.append(self.message) + + for f, lines in self.files or []: + msg.append(u'{}\n----\n{}\n'.format(f, lines)) + + return six.text_type('\n').join(msg) class InitNodeException(TestgresException): @@ -43,5 +109,5 @@ class BackupException(TestgresException): pass -class CatchUpException(TestgresException): +class InvalidOperationException(TestgresException): pass diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py new file mode 100755 index 00000000..a51af2bd --- /dev/null +++ b/testgres/impl/port_manager__generic.py @@ -0,0 +1,64 @@ +from ..operations.os_ops import OsOperations + +from ..port_manager import PortManager +from ..exceptions import PortForException + +import threading +import random +import typing + + +class PortManager__Generic(PortManager): + _os_ops: OsOperations + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] + + def __init__(self, os_ops: OsOperations): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + self._available_ports: typing.Set[int] = set(range(1024, 65535)) + self._reserved_ports: typing.Set[int] = set() + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) == set # noqa: E721t + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert not (port in self._reserved_ports) + assert port in self._available_ports + + if not self._os_ops.is_port_free(port): + continue + + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert not (port in self._available_ports) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + + assert self._guard is not None + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + assert number in self._reserved_ports + assert not (number in self._available_ports) + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert not (number in self._reserved_ports) + assert number in self._available_ports diff --git a/testgres/impl/port_manager__this_host.py b/testgres/impl/port_manager__this_host.py new file mode 100755 index 00000000..0d56f356 --- /dev/null +++ b/testgres/impl/port_manager__this_host.py @@ -0,0 +1,33 @@ +from ..port_manager import PortManager + +from .. import utils + +import threading + + +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None + sm_single_instance_guard = threading.Lock() + + @staticmethod + def get_single_instance() -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) diff --git a/testgres/logger.py b/testgres/logger.py index 820998a6..b4648f44 100644 --- a/testgres/logger.py +++ b/testgres/logger.py @@ -10,7 +10,6 @@ class TestgresLogger(threading.Thread): """ Helper class to implement reading from log files. """ - def __init__(self, node_name, log_file_name): threading.Thread.__init__(self) diff --git a/testgres/node.py b/testgres/node.py index 424d7eba..9a2f4e77 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,259 +1,817 @@ # coding: utf-8 +from __future__ import annotations -import io +import logging import os -import shutil -import six +import random +import signal import subprocess +import threading import tempfile +import platform +from queue import Queue + import time +import typing + +try: + from collections.abc import Iterable +except ImportError: + from collections import Iterable + +# we support both pg8000 and psycopg2 +try: + import psycopg2 as pglib +except ImportError: + try: + import pg8000 as pglib + except ImportError: + raise ImportError("You must have psycopg2 or pg8000 modules installed") + +from six import raise_from, iteritems, text_type -from enum import Enum -from six import raise_from +from .enums import \ + NodeStatus, \ + ProcessType, \ + DumpFormat from .cache import cached_initdb -from .config import TestgresConfig +from .config import testgres_config -from .connection import \ - NodeConnection, \ - InternalError, \ - ProgrammingError +from .connection import NodeConnection from .consts import \ DATA_DIR, \ LOGS_DIR, \ + TMP_NODE, \ + TMP_DUMP, \ PG_CONF_FILE, \ + PG_AUTO_CONF_FILE, \ HBA_CONF_FILE, \ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ UTILS_LOG_FILE, \ - DEFAULT_XLOG_METHOD + PG_CTL__STATUS__OK, \ + PG_CTL__STATUS__NODE_IS_STOPPED, \ + PG_CTL__STATUS__BAD_DATADIR \ + +from .consts import \ + MAX_LOGICAL_REPLICATION_WORKERS, \ + MAX_REPLICATION_SLOTS, \ + MAX_WORKER_PROCESSES, \ + MAX_WAL_SENDERS, \ + WAL_KEEP_SEGMENTS, \ + WAL_KEEP_SIZE + +from .decorators import \ + method_decorator, \ + positional_args_hack + +from .defaults import \ + default_dbname, \ + generate_app_name from .exceptions import \ CatchUpException, \ ExecUtilException, \ QueryException, \ StartNodeException, \ - TimeoutException + TimeoutException, \ + InitNodeException, \ + TestgresException, \ + BackupException, \ + InvalidOperationException + +from .port_manager import PortManager +from .impl.port_manager__this_host import PortManager__ThisHost +from .impl.port_manager__generic import PortManager__Generic from .logger import TestgresLogger +from .pubsub import Publication, Subscription + +from .standby import First + +from . import utils + from .utils import \ - get_bin_path, \ - file_tail, \ - pg_version_ge, \ - reserve_port, \ - release_port, \ - default_dbname, \ - default_username, \ - generate_app_name, \ - execute_utility, \ - method_decorator, \ - positional_args_hack + PgVer, \ + eprint, \ + get_bin_path2, \ + get_pg_version2, \ + execute_utility2, \ + options_string, \ + clean_on_error + +from .backup import NodeBackup +from .operations.os_ops import ConnectionParams +from .operations.os_ops import OsOperations +from .operations.local_ops import LocalOperations -class NodeStatus(Enum): +InternalError = pglib.InternalError +ProgrammingError = pglib.ProgrammingError +OperationalError = pglib.OperationalError + + +class ProcessProxy(object): """ - Status of a PostgresNode + Wrapper for psutil.Process + + Attributes: + process: wrapped psutill.Process object + ptype: instance of ProcessType """ - Running, Stopped, Uninitialized = range(3) + def __init__(self, process, ptype=None): + self.process = process + self.ptype = ptype or ProcessType.from_process(process) - # for Python 3.x - def __bool__(self): - return self.value == NodeStatus.Running.value + def __getattr__(self, name): + return getattr(self.process, name) - # for Python 2.x - __nonzero__ = __bool__ + def __repr__(self): + return '{}(ptype={}, process={})'.format( + self.__class__.__name__, + str(self.ptype), + repr(self.process)) class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, use_logging=False): + # a max number of node start attempts + _C_MAX_START_ATEMPTS = 5 + + _name: typing.Optional[str] + _port: typing.Optional[int] + _should_free_port: bool + _os_ops: OsOperations + _port_manager: PortManager + + def __init__(self, + name=None, + base_dir=None, + port: typing.Optional[int] = None, + conn_params: ConnectionParams = None, + bin_dir=None, + prefix=None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None): """ - Create a new node manually. + PostgresNode constructor. Args: name: node's application name. port: port to accept connections. base_dir: path to node's data directory. - use_logging: enable python logging. + bin_dir: path to node's binary directory. + os_ops: None or correct OS operation object. + port_manager: None or correct port manager object. """ + assert port is None or type(port) == int # noqa: E721 + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) - # public - self.host = '127.0.0.1' - self.name = name or generate_app_name() - self.port = port or reserve_port() - self.base_dir = base_dir + if conn_params is not None: + assert type(conn_params) == ConnectionParams # noqa: E721 + + raise InvalidOperationException("conn_params is deprecated, please use os_ops parameter instead.") # private - self._should_rm_dirs = base_dir is None - self._should_free_port = port is None - self._use_logging = use_logging + if os_ops is None: + self._os_ops = __class__._get_os_ops() + else: + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + pass + + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + self._pg_version = PgVer(get_pg_version2(self._os_ops, bin_dir)) + self._base_dir = base_dir + self._bin_dir = bin_dir + self._prefix = prefix self._logger = None self._master = None - # create directories if needed - self._prepare_dirs() + # basic + self._name = name or generate_app_name() + + if port is not None: + assert type(port) == int # noqa: E721 + assert port_manager is None + self._port = port + self._should_free_port = False + self._port_manager = None + else: + if port_manager is None: + self._port_manager = __class__._get_port_manager(self._os_ops) + elif os_ops is None: + raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.") + else: + assert isinstance(port_manager, PortManager) + assert self._os_ops is os_ops + self._port_manager = port_manager + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + + self._port = self._port_manager.reserve_port() # raises + assert type(self._port) == int # noqa: E721 + self._should_free_port = True + + assert type(self._port) == int # noqa: E721 + + # defaults for __exit__() + self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit + self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit + self.shutdown_max_attempts = 3 + + # NOTE: for compatibility + self.utils_log_name = self.utils_log_file + self.pg_log_name = self.pg_log_file + + # Node state + self.is_started = False def __enter__(self): return self def __exit__(self, type, value, traceback): - # stop node if necessary - self.cleanup() + # NOTE: Ctrl+C does not count! + got_exception = type is not None and type != KeyboardInterrupt - # free port if necessary - self.free_port() + c1 = self.cleanup_on_good_exit and not got_exception + c2 = self.cleanup_on_bad_exit and got_exception + + attempts = self.shutdown_max_attempts + + if c1 or c2: + self.cleanup(attempts) + else: + self._try_shutdown(attempts) + + self._release_resources() + + def __repr__(self): + return "{}(name='{}', port={}, base_dir='{}')".format( + self.__class__.__name__, + self.name, + str(self._port) if self._port is not None else "None", + self.base_dir + ) + + @staticmethod + def _get_os_ops() -> OsOperations: + if testgres_config.os_ops: + return testgres_config.os_ops + + return LocalOperations.get_single_instance() + + @staticmethod + def _get_port_manager(os_ops: OsOperations) -> PortManager: + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + if os_ops is LocalOperations.get_single_instance(): + assert utils._old_port_manager is not None + assert type(utils._old_port_manager) == PortManager__Generic # noqa: E721 + assert utils._old_port_manager._os_ops is os_ops + return PortManager__ThisHost.get_single_instance() + + # TODO: Throw the exception "Please define a port manager." ? + return PortManager__Generic(os_ops) + + def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): + assert name is None or type(name) == str # noqa: E721 + assert base_dir is None or type(base_dir) == str # noqa: E721 + + assert __class__ == PostgresNode + + if self._port_manager is None: + raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + node = PostgresNode( + name=name, + base_dir=base_dir, + bin_dir=self._bin_dir, + prefix=self._prefix, + os_ops=self._os_ops, + port_manager=self._port_manager) + + return node + + @property + def os_ops(self) -> OsOperations: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + + @property + def name(self) -> str: + if self._name is None: + raise InvalidOperationException("PostgresNode name is not defined.") + assert type(self._name) == str # noqa: E721 + return self._name + + @property + def host(self) -> str: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.host + + @property + def port(self) -> int: + if self._port is None: + raise InvalidOperationException("PostgresNode port is not defined.") + + assert type(self._port) == int # noqa: E721 + return self._port + + @property + def ssh_key(self) -> typing.Optional[str]: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.ssh_key + + @property + def pid(self): + """ + Return postmaster's PID if node is running, else 0. + """ + + self__data_dir = self.data_dir + + _params = [ + self._get_bin_path('pg_ctl'), + "-D", self__data_dir, + "status" + ] # yapf: disable + + status_code, out, error = execute_utility2( + self.os_ops, + _params, + self.utils_log_file, + verbose=True, + ignore_errors=True) + + assert type(status_code) == int # noqa: E721 + assert type(out) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + # ----------------- + if status_code == PG_CTL__STATUS__NODE_IS_STOPPED: + return 0 + + # ----------------- + if status_code == PG_CTL__STATUS__BAD_DATADIR: + return 0 + + # ----------------- + if status_code != PG_CTL__STATUS__OK: + errMsg = "Getting of a node status [data_dir is {0}] failed.".format(self__data_dir) + + raise ExecUtilException( + message=errMsg, + command=_params, + exit_code=status_code, + out=out, + error=error, + ) + + # ----------------- + assert status_code == PG_CTL__STATUS__OK + + if out == "": + __class__._throw_error__pg_ctl_returns_an_empty_string( + _params + ) + + C_PID_PREFIX = "(PID: " + + i = out.find(C_PID_PREFIX) + + if i == -1: + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + assert i > 0 + assert i < len(out) + assert len(C_PID_PREFIX) <= len(out) + assert i <= len(out) - len(C_PID_PREFIX) + + i += len(C_PID_PREFIX) + start_pid_s = i + + while True: + if i == len(out): + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + ch = out[i] + + if ch == ")": + break + + if ch.isdigit(): + i += 1 + continue + + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + assert False + + if i == start_pid_s: + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + # TODO: Let's verify a length of pid string. + + pid = int(out[start_pid_s:i]) + + if pid == 0: + __class__._throw_error__pg_ctl_returns_a_zero_pid( + out, + _params + ) + + assert pid != 0 + return pid + + @staticmethod + def _throw_error__pg_ctl_returns_an_empty_string(_params): + errLines = [] + errLines.append("Utility pg_ctl returns empty string.") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def _throw_error__pg_ctl_returns_an_unexpected_string(out, _params): + errLines = [] + errLines.append("Utility pg_ctl returns an unexpected string:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def _throw_error__pg_ctl_returns_a_zero_pid(out, _params): + errLines = [] + errLines.append("Utility pg_ctl returns a zero pid. Output string is:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @property + def auxiliary_pids(self): + """ + Returns a dict of { ProcessType : PID }. + """ + + result = {} + + for process in self.auxiliary_processes: + if process.ptype not in result: + result[process.ptype] = [] + + result[process.ptype].append(process.pid) + + return result + + @property + def auxiliary_processes(self): + """ + Returns a list of auxiliary processes. + Each process is represented by :class:`.ProcessProxy` object. + """ + def is_aux(process): + return process.ptype != ProcessType.Unknown + + return list(filter(is_aux, self.child_processes)) + + @property + def child_processes(self): + """ + Returns a list of all child processes. + Each process is represented by :class:`.ProcessProxy` object. + """ + + # get a list of postmaster's children + children = self.os_ops.get_process_children(self.pid) + + return [ProcessProxy(p) for p in children] + + @property + def source_walsender(self): + """ + Returns master's walsender feeding this replica. + """ + + sql = """ + select pid + from pg_catalog.pg_stat_replication + where application_name = %s + """ + + if self.master is None: + raise TestgresException("Node doesn't have a master") + + assert type(self.master) == PostgresNode # noqa: E721 + + # master should be on the same host + assert self.master.host == self.host + + with self.master.connect() as con: + for row in con.execute(sql, self.name): + for child in self.master.auxiliary_processes: + if child.pid == int(row[0]): + return child + + msg = "Master doesn't send WAL to {}".format(self.name) + raise TestgresException(msg) @property def master(self): return self._master @property - def data_dir(self): - return os.path.join(self.base_dir, DATA_DIR) + def base_dir(self): + if not self._base_dir: + self._base_dir = self.os_ops.mkdtemp(prefix=self._prefix or TMP_NODE) + + # NOTE: it's safe to create a new dir + if not self.os_ops.path_exists(self._base_dir): + self.os_ops.makedirs(self._base_dir) + + return self._base_dir + + @property + def bin_dir(self): + if not self._bin_dir: + self._bin_dir = os.path.dirname(get_bin_path2(self.os_ops, "pg_config")) + return self._bin_dir @property def logs_dir(self): - return os.path.join(self.base_dir, LOGS_DIR) + path = os.path.join(self.base_dir, LOGS_DIR) + + # NOTE: it's safe to create a new dir + if not self.os_ops.path_exists(path): + self.os_ops.makedirs(path) + + return path + + @property + def data_dir(self): + # NOTE: we can't run initdb without user's args + return os.path.join(self.base_dir, DATA_DIR) @property - def utils_log_name(self): + def utils_log_file(self): return os.path.join(self.logs_dir, UTILS_LOG_FILE) @property - def pg_log_name(self): + def pg_log_file(self): return os.path.join(self.logs_dir, PG_LOG_FILE) + @property + def version(self): + """ + Return PostgreSQL version for this node. + + Returns: + Instance of :class:`distutils.version.LooseVersion`. + """ + return self._pg_version + + def _try_shutdown(self, max_attempts, with_force=False): + assert type(max_attempts) == int # noqa: E721 + assert type(with_force) == bool # noqa: E721 + assert max_attempts > 0 + + attempts = 0 + + # try stopping server N times + while attempts < max_attempts: + attempts += 1 + try: + self.stop() + except ExecUtilException: + continue # one more time + except Exception: + eprint('cannot stop node {}'.format(self.name)) + break + + return # OK + + # If force stopping is enabled and PID is valid + if not with_force: + return + + node_pid = self.pid + assert node_pid is not None + assert type(node_pid) == int # noqa: E721 + + if node_pid == 0: + return + + # TODO: [2025-02-28] It is really the old ugly code. We have to rewrite it! + + ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)] + + ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) == str # noqa: E721 + + if ps_output == "": + return + + if ps_output != str(node_pid): + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + try: + eprint('Force stopping node {0} with PID {1}'.format(self.name, node_pid)) + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped - print only column pid without headers + ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) == str # noqa: E721 + + if ps_output == "": + eprint('Node {0} has been stopped successfully.'.format(self.name)) + return + + if ps_output == str(node_pid): + eprint('Failed to stop node {0}.'.format(self.name)) + return + + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + def _release_resources(self): + self.free_port() + + @staticmethod + def _throw_bugcheck__unexpected_result_of_ps(result, cmd): + assert type(result) == str # noqa: E721 + assert type(cmd) == list # noqa: E721 + errLines = [] + errLines.append("[BUG CHECK] Unexpected result of command ps:") + errLines.append(result) + errLines.append("-----") + errLines.append("Command line is {0}".format(cmd)) + raise RuntimeError("\n".join(errLines)) + def _assign_master(self, master): """NOTE: this is a private method!""" # now this node has a master self._master = master - def _create_recovery_conf(self, username): + def _create_recovery_conf(self, username, slot=None): """NOTE: this is a private method!""" # fetch master of this node master = self.master assert master is not None - # yapf: disable - conninfo = ( - u"application_name={} " - u"port={} " - u"user={} " - ).format(master.name, master.port, username) + conninfo = { + "application_name": self.name, + "port": master.port, + "user": username + } # yapf: disable # host is tricky try: import ipaddress ipaddress.ip_address(master.host) - conninfo += u"hostaddr={}".format(master.host) + conninfo["hostaddr"] = master.host except ValueError: - conninfo += u"host={}".format(master.host) + conninfo["host"] = master.host - # yapf: disable line = ( "primary_conninfo='{}'\n" - "standby_mode=on\n" - ).format(conninfo) - - self.append_conf(RECOVERY_CONF_FILE, line) - - def _prepare_dirs(self): - """NOTE: this is a private method!""" - - if not self.base_dir: - self.base_dir = tempfile.mkdtemp() - - if not os.path.exists(self.base_dir): - os.makedirs(self.base_dir) - - if not os.path.exists(self.logs_dir): - os.makedirs(self.logs_dir) + ).format(options_string(**conninfo)) # yapf: disable + # Since 12 recovery.conf had disappeared + if self.version >= PgVer('12'): + signal_name = os.path.join(self.data_dir, "standby.signal") + self.os_ops.touch(signal_name) + else: + line += "standby_mode=on\n" + + if slot: + # Connect to master for some additional actions + with master.connect(username=username) as con: + # check if slot already exists + res = con.execute( + """ + select exists ( + select from pg_catalog.pg_replication_slots + where slot_name = %s + ) + """, slot) + + if res[0][0]: + raise TestgresException( + "Slot '{}' already exists".format(slot)) + + # TODO: we should drop this slot after replica's cleanup() + con.execute( + """ + select pg_catalog.pg_create_physical_replication_slot(%s) + """, slot) + + line += "primary_slot_name={}\n".format(slot) + + if self.version >= PgVer('12'): + self.append_conf(line=line) + else: + self.append_conf(filename=RECOVERY_CONF_FILE, line=line) def _maybe_start_logger(self): - """NOTE: this is a private method!""" - - if self._use_logging: - # spawn new logger if it doesn't exist or stopped + if testgres_config.use_python_logging: + # spawn new logger if it doesn't exist or is stopped if not self._logger or not self._logger.is_alive(): - self._logger = TestgresLogger(self.name, self.pg_log_name) + self._logger = TestgresLogger(self.name, self.pg_log_file) self._logger.start() def _maybe_stop_logger(self): - """NOTE: this is a private method!""" - if self._logger: self._logger.stop() - def _format_verbose_error(self, message=None): - """NOTE: this is a private method!""" + def _collect_special_files(self): + result = [] - # list of important files + N of last lines + # list of important files + last N lines files = [ (os.path.join(self.data_dir, PG_CONF_FILE), 0), - (os.path.join(self.data_dir, HBA_CONF_FILE), 0), + (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), - (self.pg_log_name, TestgresConfig.error_log_lines) - ] - - error_text = "" - - # append message if asked to - if message: - error_text += message - error_text += '\n' * 2 + (os.path.join(self.data_dir, HBA_CONF_FILE), 0), + (self.pg_log_file, testgres_config.error_log_lines) + ] # yapf: disable for f, num_lines in files: # skip missing files - if not os.path.exists(f): + if not self.os_ops.path_exists(f): continue - with io.open(f, "rb") as _f: - if num_lines > 0: - # take last N lines of file - lines = b''.join(file_tail(_f, num_lines)).decode('utf-8') - else: - # read whole file - lines = _f.read().decode('utf-8') + file_lines = self.os_ops.readlines(f, num_lines, binary=True, encoding=None) + lines = b''.join(file_lines) - # append contents - error_text += u"{}:\n----\n{}\n".format(f, lines) + # fill list + result.append((f, lines)) - return error_text + return result - def init(self, - fsync=False, - unix_sockets=True, - allow_streaming=True, - initdb_params=[]): + def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. Args: + initdb_params: parameters for initdb (list). fsync: should this node use fsync to keep data safe? unix_sockets: should we enable UNIX sockets? allow_streaming: should this node add a hba entry for replication? - initdb_params: parameters for initdb (list). Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode` """ - # create directories if needed - self._prepare_dirs() - # initialize this PostgreSQL node - initdb_log = os.path.join(self.logs_dir, "initdb.log") - cached_initdb(self.data_dir, initdb_log, initdb_params) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + cached_initdb( + data_dir=self.data_dir, + logfile=self.utils_log_file, + os_ops=self._os_ops, + params=initdb_params, + bin_path=self.bin_dir, + cached=False) # initialize default config files - self.default_conf(fsync=fsync, - unix_sockets=unix_sockets, - allow_streaming=allow_streaming) + self.default_conf(**kwargs) return self @@ -261,6 +819,7 @@ def default_conf(self, fsync=False, unix_sockets=True, allow_streaming=True, + allow_logical=False, log_statement='all'): """ Apply default settings to this node. @@ -269,106 +828,134 @@ def default_conf(self, fsync: should this node use fsync to keep data safe? unix_sockets: should we enable UNIX sockets? allow_streaming: should this node add a hba entry for replication? + allow_logical: can this node be used as a logical replication publisher? log_statement: one of ('all', 'off', 'mod', 'ddl'). Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ postgres_conf = os.path.join(self.data_dir, PG_CONF_FILE) hba_conf = os.path.join(self.data_dir, HBA_CONF_FILE) # filter lines in hba file - with io.open(hba_conf, "r+") as conf: - # get rid of comments and blank lines - lines = [ - s for s in conf.readlines() - if len(s.strip()) > 0 and not s.startswith('#') - ] - - # write filtered lines - conf.seek(0) - conf.truncate() - conf.writelines(lines) - - # replication-related settings - if allow_streaming: - # get auth method for host or local users - def get_auth_method(t): - return next((s.split()[-1] for s in lines - if s.startswith(t)), 'trust') - - # get auth methods - auth_local = get_auth_method('local') - auth_host = get_auth_method('host') - - # yapf: disable - new_lines = [ - u"local\treplication\tall\t\t\t{}\n".format(auth_local), - u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), - u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host) - ] - - # write missing lines - for line in new_lines: - if line not in lines: - conf.write(line) + # get rid of comments and blank lines + hba_conf_file = self.os_ops.readlines(hba_conf) + lines = [ + s for s in hba_conf_file + if len(s.strip()) > 0 and not s.startswith('#') + ] + + # write filtered lines + self.os_ops.write(hba_conf, lines, truncate=True) + + # replication-related settings + if allow_streaming: + # get auth method for host or local users + def get_auth_method(t): + return next((s.split()[-1] + for s in lines if s.startswith(t)), 'trust') + + # get auth methods + auth_local = get_auth_method('local') + auth_host = get_auth_method('host') + subnet_base = ".".join(self.os_ops.host.split('.')[:-1] + ['0']) + + new_lines = [ + u"local\treplication\tall\t\t\t{}\n".format(auth_local), + u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), + u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), + u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\tall\t{}\n".format(auth_host), + u"host\treplication\tall\tall\t{}\n".format(auth_host) + ] # yapf: disable + + # write missing lines + self.os_ops.write(hba_conf, new_lines) # overwrite config file - with io.open(postgres_conf, "w") as conf: - # remove old lines - conf.truncate() - - if not fsync: - conf.write(u"fsync = off\n") - - # yapf: disable - conf.write(u"log_statement = {}\n" - u"listen_addresses = '{}'\n" - u"port = {}\n".format(log_statement, - self.host, - self.port)) - - # replication-related settings - if allow_streaming: - - # select a proper wal_level for PostgreSQL - if pg_version_ge('9.6'): - wal_level = "replica" - else: - wal_level = "hot_standby" - - # yapf: disable - max_wal_senders = 10 # default in PG 10 - wal_keep_segments = 20 # for convenience - conf.write(u"hot_standby = on\n" - u"max_wal_senders = {}\n" - u"wal_keep_segments = {}\n" - u"wal_level = {}\n".format(max_wal_senders, - wal_keep_segments, - wal_level)) - - # disable UNIX sockets if asked to - if not unix_sockets: - conf.write(u"unix_socket_directories = ''\n") + self.os_ops.write(postgres_conf, '', truncate=True) + + self.append_conf(fsync=fsync, + max_worker_processes=MAX_WORKER_PROCESSES, + log_statement=log_statement, + listen_addresses=self.host, + port=self.port) # yapf:disable + + # common replication settings + if allow_streaming or allow_logical: + self.append_conf(max_replication_slots=MAX_REPLICATION_SLOTS, + max_wal_senders=MAX_WAL_SENDERS) # yapf: disable + + # binary replication + if allow_streaming: + # select a proper wal_level for PostgreSQL + wal_level = 'replica' if self._pg_version >= PgVer('9.6') else 'hot_standby' + + if self._pg_version < PgVer('13'): + self.append_conf(hot_standby=True, + wal_keep_segments=WAL_KEEP_SEGMENTS, + wal_level=wal_level) # yapf: disable + else: + self.append_conf(hot_standby=True, + wal_keep_size=WAL_KEEP_SIZE, + wal_level=wal_level) # yapf: disable + + # logical replication + if allow_logical: + if self._pg_version < PgVer('10'): + raise InitNodeException("Logical replication is only " + "available on PostgreSQL 10 and newer") + + self.append_conf( + max_logical_replication_workers=MAX_LOGICAL_REPLICATION_WORKERS, + wal_level='logical') + + # disable UNIX sockets if asked to + if not unix_sockets: + self.append_conf(unix_socket_directories='') return self - def append_conf(self, filename, string): + @method_decorator(positional_args_hack(['filename', 'line'])) + def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): """ - Append line to a config file (i.e. postgresql.conf). + Append line to a config file. Args: - filename: name of the config file. - string: string to be appended to config. + line: string to be appended to config. + filename: config file (postgresql.conf by default). + **kwargs: named config options. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. + + Examples: + >>> append_conf(fsync=False) + >>> append_conf('log_connections = yes') + >>> append_conf(random_page_cost=1.5, fsync=True, ...) + >>> append_conf('postgresql.conf', 'synchronous_commit = off') """ + lines = [line] + + for option, value in iteritems(kwargs): + if isinstance(value, bool): + value = 'on' if value else 'off' + elif not str(value).replace('.', '', 1).isdigit(): + value = "'{}'".format(value) + if value == '*': + lines.append("{} = '*'".format(option)) + else: + # format a new config line + lines.append('{} = {}'.format(option, value)) + config_name = os.path.join(self.data_dir, filename) - with io.open(config_name, "a") as conf: - conf.write(u"".join([string, '\n'])) + conf_text = '' + for line in lines: + conf_text += text_type(line) + '\n' + self.os_ops.write(config_name, conf_text) return self @@ -377,17 +964,20 @@ def status(self): Check this node's status. Returns: - An instance of NodeStatus. + An instance of :class:`.NodeStatus`. """ try: - # yapf: disable _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path('pg_ctl'), "-D", self.data_dir, "status" - ] - execute_utility(_params, self.utils_log_name) + ] # yapf: disable + status_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: + return NodeStatus.Uninitialized + elif 'no server running' in out: + return NodeStatus.Stopped return NodeStatus.Running except ExecUtilException as e: @@ -399,29 +989,17 @@ def status(self): elif e.exit_code == 4: return NodeStatus.Uninitialized - def get_pid(self): - """ - Return postmaster's PID if node is running, else 0. - """ - - if self.status(): - with io.open(os.path.join(self.data_dir, 'postmaster.pid')) as f: - return int(f.readline()) - - # for clarity - return 0 - def get_control_data(self): """ Return contents of pg_control file. """ # this one is tricky (blame PG 9.4) - _params = [get_bin_path("pg_controldata")] - _params += ["-D"] if pg_version_ge('9.5') else [] + _params = [self._get_bin_path("pg_controldata")] + _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] _params += [self.data_dir] - data = execute_utility(_params, self.utils_log_name) + data = execute_utility2(self.os_ops, _params, self.utils_log_file) out_dict = {} @@ -431,61 +1009,181 @@ def get_control_data(self): return out_dict - def start(self, params=[]): + def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0, exec_env=None): + """ + Starts the PostgreSQL instance and then polls the instance + until it reaches the expected state (primary or replica). The state is checked + using the pg_is_in_recovery() function. + + Args: + dbname: + username: + replica: If True, waits for the instance to be in recovery (i.e., replica mode). + If False, waits for the instance to be in primary mode. Default is False. + max_attempts: + """ + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + self.start(exec_env=exec_env) + + if replica: + query = 'SELECT pg_is_in_recovery()' + else: + query = 'SELECT not pg_is_in_recovery()' + # Call poll_query_until until the expected value is returned + self.poll_query_until(query=query, + dbname=dbname, + username=username or self.os_ops.username, + suppress={InternalError, + QueryException, + ProgrammingError, + OperationalError}, + max_attempts=max_attempts) + + def start(self, params=[], wait=True, exec_env=None): """ - Start this node using pg_ctl. + Starts the PostgreSQL node using pg_ctl if node has not been started. + By default, it waits for the operation to complete before returning. + Optionally, it can return immediately without waiting for the start operation + to complete by setting the `wait` parameter to False. Args: params: additional arguments for pg_ctl. + wait: wait until operation completes. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ + assert exec_env is None or type(exec_env) == dict # noqa: E721 + assert __class__._C_MAX_START_ATEMPTS > 1 - # yapf: disable - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-l", self.pg_log_name, - "-w", # wait - "start" - ] + params + if self.is_started: + return self - try: - execute_utility(_params, self.utils_log_name) - except ExecUtilException as e: - msg = self._format_verbose_error('Cannot start node') - raise_from(StartNodeException(msg), e) + if self._port is None: + raise InvalidOperationException("Can't start PostgresNode. Port is not defined.") - self._maybe_start_logger() + assert type(self._port) == int # noqa: E721 + + _params = [self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-l", self.pg_log_file, + "-w" if wait else '-W', # --wait or --no-wait + "start"] + params # yapf: disable + + def LOCAL__start_node(): + # 'error' will be None on Windows + _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env) + assert error is None or type(error) == str # noqa: E721 + if error and 'does not exist' in error: + raise Exception(error) + + def LOCAL__raise_cannot_start_node(from_exception, msg): + assert isinstance(from_exception, Exception) + assert type(msg) == str # noqa: E721 + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), from_exception) + def LOCAL__raise_cannot_start_node__std(from_exception): + assert isinstance(from_exception, Exception) + LOCAL__raise_cannot_start_node(from_exception, 'Cannot start node') + + if not self._should_free_port: + try: + LOCAL__start_node() + except Exception as e: + LOCAL__raise_cannot_start_node__std(e) + else: + assert self._should_free_port + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + assert __class__._C_MAX_START_ATEMPTS > 1 + + log_reader = PostgresNodeLogReader(self, from_beginnig=False) + + nAttempt = 0 + timeout = 1 + while True: + assert nAttempt >= 0 + assert nAttempt < __class__._C_MAX_START_ATEMPTS + nAttempt += 1 + try: + LOCAL__start_node() + except Exception as e: + assert nAttempt > 0 + assert nAttempt <= __class__._C_MAX_START_ATEMPTS + if nAttempt == __class__._C_MAX_START_ATEMPTS: + LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts.") + + is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader) + + if not is_it_port_conflict: + LOCAL__raise_cannot_start_node__std(e) + + logging.warning( + "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) + ) + time.sleep(timeout) + timeout = min(2 * timeout, 5) + cur_port = self._port + new_port = self._port_manager.reserve_port() # can raise + try: + options = {'port': new_port} + self.set_auto_conf(options) + except: # noqa: E722 + self._port_manager.release_port(new_port) + raise + self._port = new_port + self._port_manager.release_port(cur_port) + continue + break + self._maybe_start_logger() + self.is_started = True return self - def stop(self, params=[]): + def stop(self, params=[], wait=True): """ - Stop this node using pg_ctl. + Stops the PostgreSQL node using pg_ctl if the node has been started. Args: - params: additional arguments for pg_ctl. + params: A list of additional arguments for pg_ctl. Defaults to None. + wait: If True, waits until the operation is complete. Defaults to True. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ + if not self.is_started: + return self - # yapf: disable _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, - "-w", # wait + "-w" if wait else '-W', # --wait or --no-wait "stop" - ] + params + ] + params # yapf: disable - execute_utility(_params, self.utils_log_name) + execute_utility2(self.os_ops, _params, self.utils_log_file) self._maybe_stop_logger() - + self.is_started = False return self + def kill(self, someone=None): + """ + Kills the PostgreSQL node or a specified auxiliary process if the node is running. + + Args: + someone: A key to the auxiliary process in the auxiliary_pids dictionary. + If None, the main PostgreSQL node process will be killed. Defaults to None. + """ + if self.is_started: + sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK + if someone is None: + os.kill(self.pid, sig) + else: + os.kill(self.auxiliary_pids[someone][0], sig) + self.is_started = False + def restart(self, params=[]): """ Restart this node using pg_ctl. @@ -494,23 +1192,25 @@ def restart(self, params=[]): params: additional arguments for pg_ctl. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ - # yapf: disable _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, - "-l", self.pg_log_name, + "-l", self.pg_log_file, "-w", # wait "restart" - ] + params + ] + params # yapf: disable try: - execute_utility(_params, self.utils_log_name) + error_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) + if error and 'could not start server' in error: + raise ExecUtilException except ExecUtilException as e: - msg = self._format_verbose_error('Cannot restart node') - raise_from(StartNodeException(msg), e) + msg = 'Cannot restart node' + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), e) self._maybe_start_logger() @@ -518,24 +1218,60 @@ def restart(self, params=[]): def reload(self, params=[]): """ - Reload config files using pg_ctl. + Asynchronously reload config files using pg_ctl. Args: params: additional arguments for pg_ctl. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ - # yapf: disable _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, - "-w", # wait "reload" - ] + params + ] + params # yapf: disable - execute_utility(_params, self.utils_log_name) + execute_utility2(self.os_ops, _params, self.utils_log_file) + + return self + + def promote(self, dbname=None, username=None): + """ + Promote standby instance to master using pg_ctl. For PostgreSQL versions + below 10 some additional actions required to ensure that instance + became writable and hence `dbname` and `username` parameters may be + needed. + + Returns: + This instance of :class:`.PostgresNode`. + """ + + _params = [ + self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w", # wait + "promote" + ] # yapf: disable + + execute_utility2(self.os_ops, _params, self.utils_log_file) + + # for versions below 10 `promote` is asynchronous so we need to wait + # until it actually becomes writable + if self._pg_version < PgVer('10'): + check_query = "SELECT pg_is_in_recovery()" + + self.poll_query_until(query=check_query, + expected=False, + dbname=dbname, + username=username, + max_attempts=0) # infinite + + # node becomes master itself + self._master = None + + return self def pg_ctl(self, params): """ @@ -548,58 +1284,59 @@ def pg_ctl(self, params): Stdout + stderr of pg_ctl. """ - # yapf: disable _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" # wait - ] + params + ] + params # yapf: disable - return execute_utility(_params, self.utils_log_name) + return execute_utility2(self.os_ops, _params, self.utils_log_file) def free_port(self): """ Reclaim port owned by this node. + NOTE: this method does not release manually defined port but reset it. """ + assert type(self._should_free_port) == bool # noqa: E721 - if self._should_free_port: - release_port(self.port) + if not self._should_free_port: + self._port = None + else: + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) - def cleanup(self, max_attempts=3): + port = self._port + self._should_free_port = False + self._port = None + self._port_manager.release_port(port) + + def cleanup(self, max_attempts=3, full=False, release_resources=False): """ - Stop node if needed and remove its data directory. + Stop node if needed and remove its data/logs directory. + NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? + full: clean full base dir Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ - attempts = 0 - - # try stopping server - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - print('cannot stop node {}'.format(self.name)) - - attempts += 1 + self._try_shutdown(max_attempts) - # remove directory tree if necessary - if self._should_rm_dirs: + # choose directory to be removed + if testgres_config.node_cleanup_full or full: + rm_dir = self.base_dir # everything + else: + rm_dir = self.data_dir # just data, save logs - # choose directory to be removed - if TestgresConfig.node_cleanup_full: - rm_dir = self.base_dir # everything - else: - rm_dir = self.data_dir # just data, save logs + self.os_ops.rmdirs(rm_dir, ignore_errors=False) - shutil.rmtree(rm_dir, ignore_errors=True) + if release_resources: + self._release_resources() return self @@ -609,7 +1346,10 @@ def psql(self, filename=None, dbname=None, username=None, - input=None): + input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, + **variables): """ Execute a query using psql. @@ -619,27 +1359,86 @@ def psql(self, dbname: database name to connect to. username: database user name. input: raw input to be passed. + host: an explicit host of server. + port: an explicit port of server. + **variables: vars to be set before execution. Returns: A tuple of (code, stdout, stderr). + + Examples: + >>> psql('select 1') + >>> psql('postgres', 'select 2') + >>> psql(query='select 3', ON_ERROR_STOP=1) """ - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(variables) == dict # noqa: E721 + + return self._psql( + ignore_errors=True, + query=query, + filename=filename, + dbname=dbname, + username=username, + input=input, + host=host, + port=port, + **variables + ) + + def _psql( + self, + ignore_errors, + query=None, + filename=None, + dbname=None, + username=None, + input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, + **variables): + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(variables) == dict # noqa: E721 + + # + # We do not support encoding. It may be added later. Ok? + # + if input is None: + pass + elif type(input) == bytes: # noqa: E721 + pass + else: + raise Exception("Input data must be None or bytes.") + + if host is None: + host = self.host + + if port is None: + port = self.port + + assert host is not None + assert port is not None + assert type(host) == str # noqa: E721 + assert type(port) == int # noqa: E721 - # yapf: disable psql_params = [ - get_bin_path("psql"), - "-p", str(self.port), - "-h", self.host, - "-U", username, + self._get_bin_path("psql"), + "-p", str(port), + "-h", host, + "-U", username or self.os_ops.username, + "-d", dbname or default_dbname(), "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only - "-q", # run quietly - dbname - ] + "-q" # run quietly + ] # yapf: disable + + # set variables before execution + for key, value in iteritems(variables): + psql_params.extend(["--set", '{}={}'.format(key, value)]) # select query source if query: @@ -649,46 +1448,62 @@ def psql(self, else: raise QueryException('Query or filename must be provided') - # start psql process - process = subprocess.Popen( + return self.os_ops.exec_command( psql_params, - stdin=subprocess.PIPE, + verbose=True, + input=input, + stderr=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # wait until it finishes and get stdout and stderr - out, err = process.communicate(input=input) - return process.returncode, out, err + ignore_errors=ignore_errors) @method_decorator(positional_args_hack(['dbname', 'query'])) - def safe_psql(self, - query, - dbname=None, - username=None, - input=None): + def safe_psql(self, query=None, expect_error=False, **kwargs): """ Execute a query using psql. Args: query: query to be executed. + filename: file with a query. dbname: database name to connect to. username: database user name. input: raw input to be passed. + expect_error: if True - fail if we didn't get ret + if False - fail if we got ret + + **kwargs are passed to psql(). Returns: psql's output as str. """ + assert type(kwargs) == dict # noqa: E721 + assert not ("ignore_errors" in kwargs.keys()) + assert not ("expect_error" in kwargs.keys()) - ret, out, err = self.psql(query=query, - dbname=dbname, - username=username, - input=input) - if ret: - raise QueryException((err or b'').decode('utf-8')) + # force this setting + kwargs['ON_ERROR_STOP'] = 1 + try: + ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) + except ExecUtilException as e: + if not expect_error: + raise QueryException(e.message, query) + + if type(e.error) == bytes: # noqa: E721 + return e.error.decode("utf-8") # throw + + # [2024-12-09] This situation is not expected + assert False + return e.error + + if expect_error: + raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query)) return out - def dump(self, filename=None, dbname=None, username=None): + def dump(self, + filename=None, + dbname=None, + username=None, + format=DumpFormat.Plain): """ Dump database into a file using pg_dump. NOTE: the file is not removed automatically. @@ -697,32 +1512,41 @@ def dump(self, filename=None, dbname=None, username=None): filename: database dump taken by pg_dump. dbname: database name to connect to. username: database user name. + format: format argument plain/custom/directory/tar. Returns: Path to a file containing dump. """ + # Check arguments + if not isinstance(format, DumpFormat): + try: + format = DumpFormat(format) + except ValueError: + msg = 'Invalid format "{}"'.format(format) + raise BackupException(msg) + + # Generate tmpfile or tmpdir def tmpfile(): - fd, fname = tempfile.mkstemp() - os.close(fd) + if format == DumpFormat.Directory: + fname = self.os_ops.mkdtemp(prefix=TMP_DUMP) + else: + fname = self.os_ops.mkstemp(prefix=TMP_DUMP) return fname - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() filename = filename or tmpfile() - # yapf: disable _params = [ - get_bin_path("pg_dump"), + self._get_bin_path("pg_dump"), "-p", str(self.port), "-h", self.host, "-f", filename, - "-U", username, - "-d", dbname - ] + "-U", username or self.os_ops.username, + "-d", dbname or default_dbname(), + "-F", format.value + ] # yapf: disable - execute_utility(_params, self.utils_log_name) + execute_utility2(self.os_ops, _params, self.utils_log_file) return filename @@ -731,12 +1555,29 @@ def restore(self, filename, dbname=None, username=None): Restore database from pg_dump's file. Args: - filename: database dump taken by pg_dump. + filename: database dump taken by pg_dump in custom/directory/tar formats. dbname: database name to connect to. username: database user name. """ - self.psql(filename=filename, dbname=dbname, username=username) + # Set default arguments + dbname = dbname or default_dbname() + username = username or self.os_ops.username + + _params = [ + self._get_bin_path("pg_restore"), + "-p", str(self.port), + "-h", self.host, + "-U", username, + "-d", dbname, + filename + ] # yapf: disable + + # try pg_restore if dump is binary format, and psql if not + try: + execute_utility2(self.os_ops, _params, self.utils_log_name) + except ExecUtilException: + self.psql(filename=filename, dbname=dbname, username=username) @method_decorator(positional_args_hack(['dbname', 'query'])) def poll_query_until(self, @@ -747,11 +1588,10 @@ def poll_query_until(self, sleep_time=1, expected=True, commit=True, - raise_programming_error=True, - raise_internal_error=True): + suppress=None): """ - Run a query once a second until it returs 'expected'. - Query should return single column. + Run a query once per second until it returns 'expected'. + Query should return a single value (1 row, 1 column). Args: query: query to be executed. @@ -761,45 +1601,45 @@ def poll_query_until(self, sleep_time: how much should we sleep after a failure? expected: what should be returned to break the cycle? commit: should (possible) changes be committed? - raise_programming_error: enable ProgrammingError? - raise_internal_error: enable InternalError? + suppress: a collection of exceptions to be suppressed. + + Examples: + >>> poll_query_until('select true') + >>> poll_query_until('postgres', "select now() > '01.01.2018'") + >>> poll_query_until('select false', expected=True, max_attempts=4) + >>> poll_query_until('select 1', suppress={testgres.OperationalError}) """ # sanity checks assert max_attempts >= 0 assert sleep_time > 0 - attempts = 0 while max_attempts == 0 or attempts < max_attempts: try: - res = self.execute( - dbname=dbname, - query=query, - username=username, - commit=commit) + res = self.execute(dbname=dbname, + query=query, + username=username, + commit=commit) if expected is None and res is None: return # done if res is None: - raise QueryException('Query returned None') - - if len(res) == 0: - raise QueryException('Query returned 0 rows') - - if len(res[0]) == 0: - raise QueryException('Query returned 0 columns') - - if res[0][0] == expected: + raise QueryException('Query returned None', query) + + # result set is not empty + if len(res): + if len(res[0]) == 0: + raise QueryException('Query returned 0 columns', query) + if res[0][0] == expected: + return # done + # empty result set is considered as None + elif expected is None: return # done - except ProgrammingError as e: - if raise_programming_error: - raise e - - except InternalError as e: - if raise_internal_error: - raise e + except tuple(suppress or []): + logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}") + pass # we're suppressing them time.sleep(sleep_time) attempts += 1 @@ -829,53 +1669,82 @@ def execute(self, with self.connect(dbname=dbname, username=username, - password=password) as node_con: + password=password, + autocommit=commit) as node_con: # yapf: disable res = node_con.execute(query) - if commit: - node_con.commit() - return res - def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): + def backup(self, **kwargs): """ Perform pg_basebackup. Args: username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). + base_dir: the base directory for data files and logs Returns: A smart object of type NodeBackup. """ - from .backup import NodeBackup - return NodeBackup(node=self, - username=username, - xlog_method=xlog_method) + return NodeBackup(node=self, **kwargs) - def replicate(self, - name=None, - username=None, - xlog_method=DEFAULT_XLOG_METHOD, - use_logging=False): + def replicate(self, name=None, slot=None, **kwargs): """ Create a binary replica of this node. Args: name: replica's application name. + slot: create a replication slot with the specified name. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). - use_logging: enable python logging. + base_dir: the base directory for data files and logs """ - backup = self.backup(username=username, xlog_method=xlog_method) - # transform backup into a replica - return backup.spawn_replica(name=name, - destroy=True, - use_logging=use_logging) + with clean_on_error(self.backup(**kwargs)) as backup: + return backup.spawn_replica(name=name, destroy=True, slot=slot) + + def set_synchronous_standbys(self, standbys): + """ + Set standby synchronization options. This corresponds to + `synchronous_standby_names `_ + option. Note that :meth:`~.PostgresNode.reload` or + :meth:`~.PostgresNode.restart` is needed for changes to take place. + + Args: + standbys: either :class:`.First` or :class:`.Any` object specifying + synchronization parameters or just a plain list of + :class:`.PostgresNode`s replicas which would be equivalent + to passing ``First(1, )``. For PostgreSQL 9.5 and below + it is only possible to specify a plain list of standbys as + `FIRST` and `ANY` keywords aren't supported. + + Example:: + + from testgres import get_new_node, First + + master = get_new_node().init().start() + with master.replicate().start() as standby: + master.append_conf("synchronous_commit = remote_apply") + master.set_synchronous_standbys(First(1, [standby])) + master.restart() + + """ + if self._pg_version >= PgVer('9.6'): + if isinstance(standbys, Iterable): + standbys = First(1, standbys) + else: + if isinstance(standbys, Iterable): + standbys = u", ".join(u"\"{}\"".format(r.name) + for r in standbys) + else: + raise TestgresException("Feature isn't supported in " + "Postgres 9.5 and below") + + self.append_conf("synchronous_standby_names = '{}'".format(standbys)) def catchup(self, dbname=None, username=None): """ @@ -883,36 +1752,70 @@ def catchup(self, dbname=None, username=None): """ if not self.master: - raise CatchUpException("Node doesn't have a master") + raise TestgresException("Node doesn't have a master") - if pg_version_ge('10'): - poll_lsn = "select pg_current_wal_lsn()::text" - wait_lsn = "select pg_last_wal_replay_lsn() >= '{}'::pg_lsn" + if self._pg_version >= PgVer('10'): + poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text" + wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn" else: - poll_lsn = "select pg_current_xlog_location()::text" - wait_lsn = "select pg_last_xlog_replay_location() >= '{}'::pg_lsn" + poll_lsn = "select pg_catalog.pg_current_xlog_location()::text" + wait_lsn = "select pg_catalog.pg_last_xlog_replay_location() >= '{}'::pg_lsn" try: # fetch latest LSN lsn = self.master.execute(query=poll_lsn, dbname=dbname, - username=username)[0][0] + username=username)[0][0] # yapf: disable # wait until this LSN reaches replica - self.poll_query_until( - query=wait_lsn.format(lsn), - dbname=dbname, - username=username, - max_attempts=0) # infinite + self.poll_query_until(query=wait_lsn.format(lsn), + dbname=dbname, + username=username, + max_attempts=0) # infinite except Exception as e: - raise_from(CatchUpException('Failed to catch up'), e) + raise_from(CatchUpException("Failed to catch up", poll_lsn), e) + + def publish(self, name, **kwargs): + """ + Create publication for logical replication + + Args: + pubname: publication name + tables: tables names list + dbname: database name where objects or interest are located + username: replication username + """ + return Publication(name=name, node=self, **kwargs) + + def subscribe(self, + publication, + name, + dbname=None, + username=None, + **params): + """ + Create subscription for logical replication + + Args: + name: subscription name + publication: publication object obtained from publish() + dbname: database name + username: replication username + params: subscription parameters (see documentation on `CREATE SUBSCRIPTION + `_ + for details) + """ + # yapf: disable + return Subscription(name=name, node=self, publication=publication, + dbname=dbname, username=username, **params) + # yapf: enable def pgbench(self, dbname=None, username=None, stdout=None, stderr=None, - options=[]): + options=None): """ Spawn a pgbench process. @@ -926,47 +1829,65 @@ def pgbench(self, Returns: Process created by subprocess.Popen. """ + if options is None: + options = [] - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() - # yapf: disable _params = [ - get_bin_path("pgbench"), + self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, - ] + options + "-U", username or self.os_ops.username + ] + options # yapf: disable # should be the last one _params.append(dbname) - proc = subprocess.Popen(_params, stdout=stdout, stderr=stderr) + proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, get_process=True) return proc + def pgbench_with_wait(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Do pgbench command and wait. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + """ + if options is None: + options = [] + + with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: + pgbench.wait() + return + def pgbench_init(self, **kwargs): """ Small wrapper for pgbench_run(). Sets initialize=True. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ self.pgbench_run(initialize=True, **kwargs) return self - def pgbench_run(self, - dbname=None, - username=None, - options=[], - **kwargs): + def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): """ Run pgbench with some options. - This event is logged (see self.utils_log_name). + This event is logged (see self.utils_log_file). Args: dbname: database name to connect to. @@ -974,28 +1895,26 @@ def pgbench_run(self, options: additional options for pgbench (list). **kwargs: named options for pgbench. - Examples: - pgbench_run(initialize=True, scale=2) - pgbench_run(time=10) Run pgbench --help to learn more. Returns: Stdout produced by pgbench. + + Examples: + >>> pgbench_run(initialize=True, scale=2) + >>> pgbench_run(time=10) """ - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() - # yapf: disable _params = [ - get_bin_path("pgbench"), + self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, - ] + options + "-U", username or self.os_ops.username + ] + options # yapf: disable - for key, value in six.iteritems(kwargs): + for key, value in iteritems(kwargs): # rename keys for pgbench key = key.replace('_', '-') @@ -1003,15 +1922,19 @@ def pgbench_run(self, if not isinstance(value, bool): _params.append('--{}={}'.format(key, value)) else: - assert value is True # just in case + assert value is True # just in case _params.append('--{}'.format(key)) # should be the last one _params.append(dbname) - return execute_utility(_params, self.utils_log_name) + return execute_utility2(self.os_ops, _params, self.utils_log_file) - def connect(self, dbname=None, username=None, password=None): + def connect(self, + dbname=None, + username=None, + password=None, + autocommit=False): """ Connect to a database. @@ -1019,12 +1942,535 @@ def connect(self, dbname=None, username=None, password=None): dbname: database name to connect to. username: database user name. password: user's password. + autocommit: commit each statement automatically. Also it should be + set to `True` for statements requiring to be run outside + a transaction? such as `VACUUM` or `CREATE DATABASE`. Returns: - An instance of NodeConnection. + An instance of :class:`.NodeConnection`. """ return NodeConnection(node=self, dbname=dbname, username=username, - password=password) + password=password, + autocommit=autocommit) # yapf: disable + + def table_checksum(self, table, dbname="postgres"): + con = self.connect(dbname=dbname) + + curname = "cur_" + str(random.randint(0, 2 ** 48)) + + con.execute(""" + DECLARE %s NO SCROLL CURSOR FOR + SELECT t::text FROM %s as t + """ % (curname, table)) + + que = Queue(maxsize=50) + sum = 0 + + rows = con.execute("FETCH FORWARD 2000 FROM %s" % curname) + if not rows: + return 0 + que.put(rows) + + th = None + if len(rows) == 2000: + def querier(): + try: + while True: + rows = con.execute("FETCH FORWARD 2000 FROM %s" % curname) + if not rows: + break + que.put(rows) + except Exception as e: + que.put(e) + else: + que.put(None) + + th = threading.Thread(target=querier) + th.start() + else: + que.put(None) + + while True: + rows = que.get() + if rows is None: + break + if isinstance(rows, Exception): + raise rows + # hash uses SipHash since Python3.4, therefore it is good enough + for row in rows: + sum += hash(row[0]) + + if th is not None: + th.join() + + con.execute("CLOSE %s; ROLLBACK;" % curname) + + con.close() + return sum + + def pgbench_table_checksums(self, dbname="postgres", + pgbench_tables=('pgbench_branches', + 'pgbench_tellers', + 'pgbench_accounts', + 'pgbench_history') + ): + return {(table, self.table_checksum(table, dbname)) + for table in pgbench_tables} + + def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): + """ + Update or remove configuration options in the specified configuration file, + updates the options specified in the options dictionary, removes any options + specified in the rm_options set, and writes the updated configuration back to + the file. + + Args: + options (dict): A dictionary containing the options to update or add, + with the option names as keys and their values as values. + config (str, optional): The name of the configuration file to update. + Defaults to 'postgresql.auto.conf'. + rm_options (set, optional): A set containing the names of the options to remove. + Defaults to an empty set. + """ + # parse postgresql.auto.conf + path = os.path.join(self.data_dir, config) + + lines = self.os_ops.readlines(path) + current_options = {} + current_directives = [] + for line in lines: + + # ignore comments + if line.startswith('#'): + continue + + if line.strip() == '': + continue + + if line.startswith('include'): + current_directives.append(line) + continue + + name, var = line.partition('=')[::2] + name = name.strip() + + # Remove options specified in rm_options list + if name in rm_options: + continue + + current_options[name] = var + + for option in options: + assert type(option) == str # noqa: E721 + assert option != "" + assert option.strip() == option + + value = options[option] + valueType = type(value) + + if valueType == str: + value = __class__._escape_config_value(value) + elif valueType == bool: + value = "on" if value else "off" + + current_options[option] = value + + auto_conf = '' + for option in current_options: + auto_conf += option + " = " + str(current_options[option]) + "\n" + + for directive in current_directives: + auto_conf += directive + "\n" + + self.os_ops.write(path, auto_conf, truncate=True) + + def upgrade_from(self, old_node, options=None, expect_error=False): + """ + Upgrade this node from an old node using pg_upgrade. + + Args: + old_node: An instance of PostgresNode representing the old node. + """ + if not os.path.exists(old_node.data_dir): + raise Exception("Old node must be initialized") + + if not os.path.exists(self.data_dir): + self.init() + + if not options: + options = [] + + pg_upgrade_binary = self._get_bin_path("pg_upgrade") + + if not os.path.exists(pg_upgrade_binary): + raise Exception("pg_upgrade does not exist in the new node's binary path") + + upgrade_command = [ + pg_upgrade_binary, + "--old-bindir", old_node.bin_dir, + "--new-bindir", self.bin_dir, + "--old-datadir", old_node.data_dir, + "--new-datadir", self.data_dir, + "--old-port", str(old_node.port), + "--new-port", str(self.port) + ] + upgrade_command += options + + return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) + + def _get_bin_path(self, filename): + if self.bin_dir: + bin_path = os.path.join(self.bin_dir, filename) + else: + bin_path = get_bin_path2(self.os_ops, filename) + return bin_path + + def _escape_config_value(value): + assert type(value) == str # noqa: E721 + + result = "'" + + for ch in value: + if ch == "'": + result += "\\'" + elif ch == "\n": + result += "\\n" + elif ch == "\r": + result += "\\r" + elif ch == "\t": + result += "\\t" + elif ch == "\b": + result += "\\b" + elif ch == "\\": + result += "\\\\" + else: + result += ch + + result += "'" + return result + + +class PostgresNodeLogReader: + class LogInfo: + position: int + + def __init__(self, position: int): + self.position = position + + # -------------------------------------------------------------------- + class LogDataBlock: + _file_name: str + _position: int + _data: str + + def __init__( + self, + file_name: str, + position: int, + data: str + ): + assert type(file_name) == str # noqa: E721 + assert type(position) == int # noqa: E721 + assert type(data) == str # noqa: E721 + assert file_name != "" + assert position >= 0 + self._file_name = file_name + self._position = position + self._data = data + + @property + def file_name(self) -> str: + assert type(self._file_name) == str # noqa: E721 + assert self._file_name != "" + return self._file_name + + @property + def position(self) -> int: + assert type(self._position) == int # noqa: E721 + assert self._position >= 0 + return self._position + + @property + def data(self) -> str: + assert type(self._data) == str # noqa: E721 + return self._data + + # -------------------------------------------------------------------- + _node: PostgresNode + _logs: typing.Dict[str, LogInfo] + + # -------------------------------------------------------------------- + def __init__(self, node: PostgresNode, from_beginnig: bool): + assert node is not None + assert isinstance(node, PostgresNode) + assert type(from_beginnig) == bool # noqa: E721 + + self._node = node + + if from_beginnig: + self._logs = dict() + else: + self._logs = self._collect_logs() + + assert type(self._logs) == dict # noqa: E721 + return + + def read(self) -> typing.List[LogDataBlock]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + cur_logs: typing.Dict[__class__.LogInfo] = self._collect_logs() + assert cur_logs is not None + assert type(cur_logs) == dict # noqa: E721 + + assert type(self._logs) == dict # noqa: E721 + + result = list() + + for file_name, cur_log_info in cur_logs.items(): + assert type(file_name) == str # noqa: E721 + assert type(cur_log_info) == __class__.LogInfo # noqa: E721 + + read_pos = 0 + + if file_name in self._logs.keys(): + prev_log_info = self._logs[file_name] + assert type(prev_log_info) == __class__.LogInfo # noqa: E721 + read_pos = prev_log_info.position # the previous size + + file_content_b = self._node.os_ops.read_binary(file_name, read_pos) + assert type(file_content_b) == bytes # noqa: E721 + + # + # A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol. + # + file_content_s = file_content_b.decode() + assert type(file_content_s) == str # noqa: E721 + + next_read_pos = read_pos + len(file_content_b) + + # It is a research/paranoja check. + # When we will process partial UTF-8 symbol, it must be adjusted. + assert cur_log_info.position <= next_read_pos + + cur_log_info.position = next_read_pos + + block = __class__.LogDataBlock( + file_name, + read_pos, + file_content_s + ) + + result.append(block) + + # A new check point + self._logs = cur_logs + + return result + + def _collect_logs(self) -> typing.Dict[LogInfo]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + files = [ + self._node.pg_log_file + ] # yapf: disable + + result = dict() + + for f in files: + assert type(f) == str # noqa: E721 + + # skip missing files + if not self._node.os_ops.path_exists(f): + continue + + file_size = self._node.os_ops.get_file_size(f) + assert type(file_size) == int # noqa: E721 + assert file_size >= 0 + + result[f] = __class__.LogInfo(file_size) + + return result + + +class PostgresNodeUtils: + @staticmethod + def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: + assert type(log_reader) == PostgresNodeLogReader # noqa: E721 + + blocks = log_reader.read() + assert type(blocks) == list # noqa: E721 + + for block in blocks: + assert type(block) == PostgresNodeLogReader.LogDataBlock # noqa: E721 + + if 'Is another postmaster already running on port' in block.data: + return True + + return False + + +class NodeApp: + + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + + if test_path: + if os.path.isabs(test_path): + self.test_path = test_path + else: + self.test_path = os.path.join(os_ops.cwd(), test_path) + else: + self.test_path = os_ops.cwd() + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] + self.os_ops = os_ops + + def make_empty( + self, + base_dir=None, + port=None, + bin_dir=None): + real_base_dir = os.path.join(self.test_path, base_dir) + self.os_ops.rmdirs(real_base_dir, ignore_errors=True) + self.os_ops.makedirs(real_base_dir) + + node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) + node.should_rm_dirs = True + self.nodes_to_cleanup.append(node) + + return node + + def make_simple( + self, + base_dir=None, + port=None, + set_replication=False, + ptrack_enable=False, + initdb_params=[], + pg_options={}, + checksum=True, + bin_dir=None): + assert type(pg_options) == dict # noqa: E721 + + if checksum and '--data-checksums' not in initdb_params: + initdb_params.append('--data-checksums') + node = self.make_empty(base_dir, port, bin_dir=bin_dir) + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # set major version + pg_version_file = self.os_ops.read(os.path.join(node.data_dir, 'PG_VERSION')) + node.major_version_str = str(pg_version_file.rstrip()) + node.major_version = float(node.major_version_str) + + # Set default parameters + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # unix_socket_directories will be defined later + } + + # Allow replication in pg_hba.conf + if set_replication: + options['max_wal_senders'] = 10 + + if ptrack_enable: + options['ptrack.map_size'] = '1' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '12' + + # Apply given parameters + for option_name, option_value in iteritems(pg_options): + options[option_name] = option_value + + # Define delayed propertyes + if not ("unix_socket_directories" in options.keys()): + options["unix_socket_directories"] = __class__._gettempdir_for_socket() + + # Set config values + node.set_auto_conf(options) + + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + + return node + + @staticmethod + def _gettempdir_for_socket(): + platform_system_name = platform.system().lower() + + if platform_system_name == "windows": + return __class__._gettempdir() + + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + + return "/tmp" + + @staticmethod + def _gettempdir(): + v = tempfile.gettempdir() + + # + # Paranoid checks + # + if type(v) != str: # noqa: E721 + __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + + if not os.path.exists(v): + __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + @staticmethod + def _raise_bugcheck(msg): + assert type(msg) == str # noqa: E721 + assert msg != "" + raise Exception("[BUG CHECK] " + msg) diff --git a/testgres/operations/__init__.py b/testgres/operations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py new file mode 100644 index 00000000..ebbf0f73 --- /dev/null +++ b/testgres/operations/helpers.py @@ -0,0 +1,55 @@ +import locale + + +class Helpers: + @staticmethod + def _make_get_default_encoding_func(): + # locale.getencoding is added in Python 3.11 + if hasattr(locale, 'getencoding'): + return locale.getencoding + + # It must exist + return locale.getpreferredencoding + + # Prepared pointer on function to get a name of system codepage + _get_default_encoding_func = _make_get_default_encoding_func.__func__() + + @staticmethod + def GetDefaultEncoding(): + # + # Original idea/source was: + # + # def os_ops.get_default_encoding(): + # if not hasattr(locale, 'getencoding'): + # locale.getencoding = locale.getpreferredencoding + # return locale.getencoding() or 'UTF-8' + # + + assert __class__._get_default_encoding_func is not None + + r = __class__._get_default_encoding_func() + + if r: + assert r is not None + assert type(r) == str # noqa: E721 + assert r != "" + return r + + # Is it an unexpected situation? + return 'UTF-8' + + @staticmethod + def PrepareProcessInput(input, encoding): + if not input: + return None + + if type(input) == str: # noqa: E721 + if encoding is None: + return input.encode(__class__.GetDefaultEncoding()) + + assert type(encoding) == str # noqa: E721 + return input.encode(encoding) + + # It is expected! + assert type(input) == bytes # noqa: E721 + return input diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py new file mode 100644 index 00000000..ccf1ab82 --- /dev/null +++ b/testgres/operations/local_ops.py @@ -0,0 +1,537 @@ +import getpass +import logging +import os +import shutil +import stat +import subprocess +import tempfile +import time +import socket + +import psutil +import typing +import threading + +from ..exceptions import ExecUtilException +from ..exceptions import InvalidOperationException +from .os_ops import ConnectionParams, OsOperations, get_default_encoding +from .raise_error import RaiseError +from .helpers import Helpers + +try: + from shutil import which as find_executable + from shutil import rmtree +except ImportError: + from distutils.spawn import find_executable + from distutils import rmtree + +CMD_TIMEOUT_SEC = 60 + + +class LocalOperations(OsOperations): + sm_single_instance: OsOperations = None + sm_single_instance_guard = threading.Lock() + + def __init__(self, conn_params=None): + if conn_params is None: + conn_params = ConnectionParams() + super(LocalOperations, self).__init__(conn_params.username) + self.conn_params = conn_params + self.host = conn_params.host + self.ssh_key = None + self.remote = False + self.username = conn_params.username or getpass.getuser() + + @staticmethod + def get_single_instance() -> OsOperations: + assert __class__ == LocalOperations + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + @staticmethod + def _process_output(encoding, temp_file_path): + """Process the output of a command from a temporary file.""" + with open(temp_file_path, 'rb') as temp_file: + output = temp_file.read() + if encoding: + output = output.decode(encoding) + return output, None # In Windows stderr writing in stdout + + def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + # TODO: why don't we use the data from input? + + extParams: typing.Dict[str, str] = dict() + + if exec_env is None: + pass + elif len(exec_env) == 0: + pass + else: + env = os.environ.copy() + assert type(env) == dict # noqa: E721 + for v in exec_env.items(): + assert type(v) == tuple # noqa: E721 + assert len(v) == 2 + assert type(v[0]) == str # noqa: E721 + assert v[0] != "" + + if v[1] is None: + env.pop(v[0], None) + else: + assert type(v[1]) == str # noqa: E721 + env[v[0]] = v[1] + + extParams["env"] = env + + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: + stdout = temp_file + stderr = subprocess.STDOUT + process = subprocess.Popen( + cmd, + shell=shell, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout, + stderr=stderr, + **extParams, + ) + if get_process: + return process, None, None + temp_file_path = temp_file.name + + # Wait process finished + process.wait() + + output, error = self._process_output(encoding, temp_file_path) + return process, output, error + + def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + + extParams: typing.Dict[str, str] = dict() + + if exec_env is None: + pass + elif len(exec_env) == 0: + pass + else: + env = os.environ.copy() + assert type(env) == dict # noqa: E721 + for v in exec_env.items(): + assert type(v) == tuple # noqa: E721 + assert len(v) == 2 + assert type(v[0]) == str # noqa: E721 + assert v[0] != "" + + if v[1] is None: + env.pop(v[0], None) + else: + assert type(v[1]) == str # noqa: E721 + env[v[0]] = v[1] + + extParams["env"] = env + + process = subprocess.Popen( + cmd, + shell=shell, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout or subprocess.PIPE, + stderr=stderr or subprocess.PIPE, + **extParams + ) + assert not (process is None) + if get_process: + return process, None, None + try: + output, error = process.communicate(input=input_prepared, timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + + assert type(output) == bytes # noqa: E721 + assert type(error) == bytes # noqa: E721 + + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error + + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + """Execute a command and return the process and its output.""" + if os.name == 'nt' and stdout is None: # Windows + method = __class__._run_command__nt + else: # Other OS + method = __class__._run_command__generic + + return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) + + def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, + text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, + ignore_errors=False, exec_env=None): + """ + Execute a command in a subprocess and handle the output based on the provided parameters. + """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) + if get_process: + return process + + if expect_error: + if process.returncode == 0: + raise InvalidOperationException("We expected an execution error.") + elif ignore_errors: + pass + elif process.returncode == 0: + pass + else: + assert not expect_error + assert not ignore_errors + assert process.returncode != 0 + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=process.returncode, + msg_arg=error or output, + error=error, + out=output) + + if verbose: + return process.returncode, output, error + + return output + + # Environment setup + def environ(self, var_name): + return os.environ.get(var_name) + + def cwd(self): + return os.getcwd() + + def find_executable(self, executable): + return find_executable(executable) + + def is_executable(self, file): + # Check if the file is executable + assert stat.S_IXUSR != 0 + return (os.stat(file).st_mode & stat.S_IXUSR) == stat.S_IXUSR + + def set_env(self, var_name, var_val): + # Check if the directory is already in PATH + os.environ[var_name] = var_val + + def get_name(self): + return os.name + + # Work with dirs + def makedirs(self, path, remove_existing=False): + if remove_existing: + shutil.rmtree(path, ignore_errors=True) + try: + os.makedirs(path) + except FileExistsError: + pass + + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + os.mkdir(path) + + # [2025-02-03] Old name of parameter attempts is "retries". + def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): + """ + Removes a directory and its contents, retrying on failure. + + :param path: Path to the directory. + :param ignore_errors: If True, ignore errors. + :param retries: Number of attempts to remove the directory. + :param delay: Delay between attempts in seconds. + """ + assert type(path) == str # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + assert type(attempts) == int # noqa: E721 + assert type(delay) == int or type(delay) == float # noqa: E721 + assert attempts > 0 + assert delay >= 0 + + attempt = 0 + while True: + assert attempt < attempts + attempt += 1 + try: + rmtree(path) + except FileNotFoundError: + pass + except Exception as e: + if attempt < attempt: + errMsg = "Failed to remove directory {0} on attempt {1} ({2}): {3}".format( + path, attempt, type(e).__name__, e + ) + logging.warning(errMsg) + time.sleep(delay) + continue + + assert attempt == attempts + if not ignore_errors: + raise + + return False + + # OK! + return True + + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + os.rmdir(path) + + def listdir(self, path): + return os.listdir(path) + + def path_exists(self, path): + return os.path.exists(path) + + @property + def pathsep(self): + os_name = self.get_name() + if os_name == "posix": + pathsep = ":" + elif os_name == "nt": + pathsep = ";" + else: + raise Exception("Unsupported operating system: {}".format(os_name)) + return pathsep + + def mkdtemp(self, prefix=None): + return tempfile.mkdtemp(prefix='{}'.format(prefix)) + + def mkstemp(self, prefix=None): + fd, filename = tempfile.mkstemp(prefix=prefix) + os.close(fd) # Close the file descriptor immediately after creating the file + return filename + + def copytree(self, src, dst): + return shutil.copytree(src, dst) + + # Work with files + def write(self, filename, data, truncate=False, binary=False, read_and_write=False): + """ + Write data to a file locally + Args: + filename: The file path where the data will be written. + data: The data to be written to the file. + truncate: If True, the file will be truncated before writing ('w' option); + if False (default), data will be appended ('a' option). + binary: If True, the data will be written in binary mode ('b' option); + if False (default), the data will be written in text mode. + read_and_write: If True, the file will be opened with read and write permissions ('+' option); + if False (default), only write permission will be used. + """ + if isinstance(data, bytes) or isinstance(data, list) and all(isinstance(item, bytes) for item in data): + binary = True + + mode = "w" if truncate else "a" + + if read_and_write: + mode += "+" + + # If it is a bytes str or list + if binary: + mode += "b" + + assert type(mode) == str # noqa: E721 + assert mode != "" + + with open(filename, mode) as file: + if isinstance(data, list): + data2 = [__class__._prepare_line_to_write(s, binary) for s in data] + file.writelines(data2) + else: + data2 = __class__._prepare_data_to_write(data, binary) + file.write(data2) + + @staticmethod + def _prepare_line_to_write(data, binary): + data = __class__._prepare_data_to_write(data, binary) + + if binary: + assert type(data) == bytes # noqa: E721 + return data.rstrip(b'\n') + b'\n' + + assert type(data) == str # noqa: E721 + return data.rstrip('\n') + '\n' + + @staticmethod + def _prepare_data_to_write(data, binary): + if isinstance(data, bytes): + return data if binary else data.decode() + + if isinstance(data, str): + return data if not binary else data.encode() + + raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) + + def touch(self, filename): + """ + Create a new file or update the access and modification times of an existing file. + Args: + filename (str): The name of the file to touch. + + This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. + """ + # cross-python touch(). It is vulnerable to races, but who cares? + with open(filename, "a"): + os.utime(filename, None) + + def read(self, filename, encoding=None, binary=False): + assert type(filename) == str # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + + if binary: + if encoding is not None: + raise InvalidOperationException("Enconding is not allowed for read binary operation") + + return self._read__binary(filename) + + # python behavior + assert (None or "abc") == "abc" + assert ("" or "abc") == "abc" + + return self._read__text_with_encoding(filename, encoding or get_default_encoding()) + + def _read__text_with_encoding(self, filename, encoding): + assert type(filename) == str # noqa: E721 + assert type(encoding) == str # noqa: E721 + with open(filename, mode='r', encoding=encoding) as file: # open in a text mode + content = file.read() + assert type(content) == str # noqa: E721 + return content + + def _read__binary(self, filename): + assert type(filename) == str # noqa: E721 + with open(filename, 'rb') as file: # open in a binary mode + content = file.read() + assert type(content) == bytes # noqa: E721 + return content + + def readlines(self, filename, num_lines=0, binary=False, encoding=None): + """ + Read lines from a local file. + If num_lines is greater than 0, only the last num_lines lines will be read. + """ + assert type(num_lines) == int # noqa: E721 + assert type(filename) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + assert num_lines >= 0 + + if binary: + assert encoding is None + pass + elif encoding is None: + encoding = get_default_encoding() + assert type(encoding) == str # noqa: E721 + else: + assert type(encoding) == str # noqa: E721 + pass + + mode = 'rb' if binary else 'r' + if num_lines == 0: + with open(filename, mode, encoding=encoding) as file: # open in binary mode + return file.readlines() + else: + bufsize = 8192 + buffers = 1 + + with open(filename, mode, encoding=encoding) as file: # open in binary mode + file.seek(0, os.SEEK_END) + end_pos = file.tell() + + while True: + offset = max(0, end_pos - bufsize * buffers) + file.seek(offset, os.SEEK_SET) + pos = file.tell() + lines = file.readlines() + cur_lines = len(lines) + + if cur_lines >= num_lines or pos == 0: + return lines[-num_lines:] # get last num_lines from lines + + buffers = int( + buffers * max(2, int(num_lines / max(cur_lines, 1))) + ) # Adjust buffer size + + def read_binary(self, filename, offset): + assert type(filename) == str # noqa: E721 + assert type(offset) == int # noqa: E721 + + if offset < 0: + raise ValueError("Negative 'offset' is not supported.") + + with open(filename, 'rb') as file: # open in a binary mode + file.seek(offset, os.SEEK_SET) + r = file.read() + assert type(r) == bytes # noqa: E721 + return r + + def isfile(self, remote_file): + return os.path.isfile(remote_file) + + def isdir(self, dirname): + return os.path.isdir(dirname) + + def get_file_size(self, filename): + assert filename is not None + assert type(filename) == str # noqa: E721 + return os.path.getsize(filename) + + def remove_file(self, filename): + return os.remove(filename) + + # Processes control + def kill(self, pid, signal, expect_error=False): + # Kill the process + cmd = "kill -{} {}".format(signal, pid) + return self.exec_command(cmd, expect_error=expect_error) + + def get_pid(self): + # Get current process id + return os.getpid() + + def get_process_children(self, pid): + assert type(pid) == int # noqa: E721 + return psutil.Process(pid).children() + + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", number)) + return True + except OSError: + return False + + def get_tempdir(self) -> str: + r = tempfile.gettempdir() + assert r is not None + assert type(r) == str # noqa: E721 + assert os.path.exists(r) + return r diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py new file mode 100644 index 00000000..45e4f71c --- /dev/null +++ b/testgres/operations/os_ops.py @@ -0,0 +1,135 @@ +import getpass +import locale + + +class ConnectionParams: + def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): + self.host = host + self.port = port + self.ssh_key = ssh_key + self.username = username + + +def get_default_encoding(): + if not hasattr(locale, 'getencoding'): + locale.getencoding = locale.getpreferredencoding + return locale.getencoding() or 'UTF-8' + + +class OsOperations: + def __init__(self, username=None): + self.ssh_key = None + self.username = username or getpass.getuser() + + # Command execution + def exec_command(self, cmd, **kwargs): + raise NotImplementedError() + + # Environment setup + def environ(self, var_name): + raise NotImplementedError() + + def cwd(self): + raise NotImplementedError() + + def find_executable(self, executable): + raise NotImplementedError() + + def is_executable(self, file): + # Check if the file is executable + raise NotImplementedError() + + def set_env(self, var_name, var_val): + # Check if the directory is already in PATH + raise NotImplementedError() + + def get_user(self): + return self.username + + def get_name(self): + raise NotImplementedError() + + # Work with dirs + def makedirs(self, path, remove_existing=False): + raise NotImplementedError() + + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + + def rmdirs(self, path, ignore_errors=True): + raise NotImplementedError() + + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + + def listdir(self, path): + raise NotImplementedError() + + def path_exists(self, path): + raise NotImplementedError() + + @property + def pathsep(self): + raise NotImplementedError() + + def mkdtemp(self, prefix=None): + raise NotImplementedError() + + def mkstemp(self, prefix=None): + raise NotImplementedError() + + def copytree(self, src, dst): + raise NotImplementedError() + + # Work with files + def write(self, filename, data, truncate=False, binary=False, read_and_write=False): + raise NotImplementedError() + + def touch(self, filename): + raise NotImplementedError() + + def read(self, filename, encoding, binary): + raise NotImplementedError() + + def readlines(self, filename): + raise NotImplementedError() + + def read_binary(self, filename, offset): + assert type(filename) == str # noqa: E721 + assert type(offset) == int # noqa: E721 + assert offset >= 0 + raise NotImplementedError() + + def isfile(self, remote_file): + raise NotImplementedError() + + def isdir(self, dirname): + raise NotImplementedError() + + def get_file_size(self, filename): + raise NotImplementedError() + + def remove_file(self, filename): + assert type(filename) == str # noqa: E721 + raise NotImplementedError() + + # Processes control + def kill(self, pid, signal): + # Kill the process + raise NotImplementedError() + + def get_pid(self): + # Get current process id + raise NotImplementedError() + + def get_process_children(self, pid): + raise NotImplementedError() + + def is_port_free(self, number: int): + assert type(number) == int # noqa: E721 + raise NotImplementedError() + + def get_tempdir(self) -> str: + raise NotImplementedError() diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py new file mode 100644 index 00000000..0d14be5a --- /dev/null +++ b/testgres/operations/raise_error.py @@ -0,0 +1,57 @@ +from ..exceptions import ExecUtilException +from .helpers import Helpers + + +class RaiseError: + @staticmethod + def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): + assert type(exit_code) == int # noqa: E721 + + msg_arg_s = __class__._TranslateDataIntoString(msg_arg) + assert type(msg_arg_s) == str # noqa: E721 + + msg_arg_s = msg_arg_s.strip() + if msg_arg_s == "": + msg_arg_s = "#no_error_message" + + message = "Utility exited with non-zero code (" + str(exit_code) + "). Error: `" + msg_arg_s + "`" + raise ExecUtilException( + message=message, + command=cmd, + exit_code=exit_code, + out=out, + error=error) + + @staticmethod + def CommandExecutionError(cmd, exit_code, message, error, out): + assert type(exit_code) == int # noqa: E721 + assert type(message) == str # noqa: E721 + assert message != "" + + raise ExecUtilException( + message=message, + command=cmd, + exit_code=exit_code, + out=out, + error=error) + + @staticmethod + def _TranslateDataIntoString(data): + if data is None: + return "" + + if type(data) == bytes: # noqa: E721 + return __class__._TranslateDataIntoString__FromBinary(data) + + return str(data) + + @staticmethod + def _TranslateDataIntoString__FromBinary(data): + assert type(data) == bytes # noqa: E721 + + try: + return data.decode(Helpers.GetDefaultEncoding()) + except UnicodeDecodeError: + pass + + return "#cannot_decode_text" diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py new file mode 100644 index 00000000..a478b453 --- /dev/null +++ b/testgres/operations/remote_ops.py @@ -0,0 +1,779 @@ +import getpass +import os +import platform +import subprocess +import tempfile +import io +import logging +import typing + +from ..exceptions import ExecUtilException +from ..exceptions import InvalidOperationException +from .os_ops import OsOperations, ConnectionParams, get_default_encoding +from .raise_error import RaiseError +from .helpers import Helpers + +error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] + + +class PsUtilProcessProxy: + def __init__(self, ssh, pid): + assert isinstance(ssh, RemoteOperations) + assert type(pid) == int # noqa: E721 + self.ssh = ssh + self.pid = pid + + def kill(self): + assert isinstance(self.ssh, RemoteOperations) + assert type(self.pid) == int # noqa: E721 + command = ["kill", str(self.pid)] + self.ssh.exec_command(command, encoding=get_default_encoding()) + + def cmdline(self): + assert isinstance(self.ssh, RemoteOperations) + assert type(self.pid) == int # noqa: E721 + command = ["ps", "-p", str(self.pid), "-o", "cmd", "--no-headers"] + output = self.ssh.exec_command(command, encoding=get_default_encoding()) + assert type(output) == str # noqa: E721 + cmdline = output.strip() + # TODO: This code work wrong if command line contains quoted values. Yes? + return cmdline.split() + + +class RemoteOperations(OsOperations): + def __init__(self, conn_params: ConnectionParams): + if not platform.system().lower() == "linux": + raise EnvironmentError("Remote operations are supported only on Linux!") + + super().__init__(conn_params.username) + self.conn_params = conn_params + self.host = conn_params.host + self.port = conn_params.port + self.ssh_key = conn_params.ssh_key + self.ssh_args = [] + if self.ssh_key: + self.ssh_args += ["-i", self.ssh_key] + if self.port: + self.ssh_args += ["-p", self.port] + self.remote = True + self.username = conn_params.username or getpass.getuser() + self.ssh_dest = f"{self.username}@{self.host}" if conn_params.username else self.host + + def __enter__(self): + return self + + def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, + encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, + stderr=None, get_process=None, timeout=None, ignore_errors=False, + exec_env=None): + """ + Execute a command in the SSH session. + Args: + - cmd (str): The command to be executed. + """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + + if type(cmd) == str: # noqa: E721 + cmd_s = cmd + elif type(cmd) == list: # noqa: E721 + cmd_s = subprocess.list2cmdline(cmd) + else: + raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + + assert type(cmd_s) == str # noqa: E721 + + cmd_items = __class__._make_exec_env_list(exec_env=exec_env) + cmd_items.append(cmd_s) + + env_cmd_s = ';'.join(cmd_items) + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [env_cmd_s] + + process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert not (process is None) + if get_process: + return process + + try: + output, error = process.communicate(input=input_prepared, timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + + assert type(output) == bytes # noqa: E721 + assert type(error) == bytes # noqa: E721 + + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + + if expect_error: + if process.returncode == 0: + raise InvalidOperationException("We expected an execution error.") + elif ignore_errors: + pass + elif process.returncode == 0: + pass + else: + assert not expect_error + assert not ignore_errors + assert process.returncode != 0 + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=process.returncode, + msg_arg=error, + error=error, + out=output) + + if verbose: + return process.returncode, output, error + + return output + + # Environment setup + def environ(self, var_name: str) -> str: + """ + Get the value of an environment variable. + Args: + - var_name (str): The name of the environment variable. + """ + cmd = "echo ${}".format(var_name) + return self.exec_command(cmd, encoding=get_default_encoding()).strip() + + def cwd(self): + cmd = 'pwd' + return self.exec_command(cmd, encoding=get_default_encoding()).rstrip() + + def find_executable(self, executable): + search_paths = self.environ("PATH") + if not search_paths: + return None + + search_paths = search_paths.split(self.pathsep) + for path in search_paths: + remote_file = os.path.join(path, executable) + if self.isfile(remote_file): + return remote_file + + return None + + def is_executable(self, file): + # Check if the file is executable + command = ["test", "-x", file] + + exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return True + + if exit_status == 1: + return False + + errMsg = "Test operation returns an unknown result code: {0}. File name is [{1}].".format( + exit_status, + file) + + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exit_status, + message=errMsg, + error=error, + out=output + ) + + def set_env(self, var_name: str, var_val: str): + """ + Set the value of an environment variable. + Args: + - var_name (str): The name of the environment variable. + - var_val (str): The value to be set for the environment variable. + """ + return self.exec_command("export {}={}".format(var_name, var_val)) + + def get_name(self): + cmd = 'python3 -c "import os; print(os.name)"' + return self.exec_command(cmd, encoding=get_default_encoding()).strip() + + # Work with dirs + def makedirs(self, path, remove_existing=False): + """ + Create a directory in the remote server. + Args: + - path (str): The path to the directory to be created. + - remove_existing (bool): If True, the existing directory at the path will be removed. + """ + if remove_existing: + cmd = "rm -rf {} && mkdir -p {}".format(path, path) + else: + cmd = "mkdir -p {}".format(path) + try: + exit_status, result, error = self.exec_command(cmd, verbose=True) + except ExecUtilException as e: + raise Exception("Couldn't create dir {} because of error {}".format(path, e.message)) + if exit_status != 0: + raise Exception("Couldn't create dir {} because of error {}".format(path, error)) + return result + + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["mkdir", path] + self.exec_command(cmd) + + def rmdirs(self, path, ignore_errors=True): + """ + Remove a directory in the remote server. + Args: + - path (str): The path to the directory to be removed. + - ignore_errors (bool): If True, do not raise error if directory does not exist. + """ + assert type(path) == str # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + + # ENOENT = 2 - No such file or directory + # ENOTDIR = 20 - Not a directory + + cmd1 = [ + "if", "[", "-d", path, "]", ";", + "then", "rm", "-rf", path, ";", + "elif", "[", "-e", path, "]", ";", + "then", "{", "echo", "cannot remove '" + path + "': it is not a directory", ">&2", ";", "exit", "20", ";", "}", ";", + "else", "{", "echo", "directory '" + path + "' does not exist", ">&2", ";", "exit", "2", ";", "}", ";", + "fi" + ] + + cmd2 = ["sh", "-c", subprocess.list2cmdline(cmd1)] + + try: + self.exec_command(cmd2, encoding=Helpers.GetDefaultEncoding()) + except ExecUtilException as e: + if e.exit_code == 2: # No such file or directory + return True + + if not ignore_errors: + raise + + errMsg = "Failed to remove directory {0} ({1}): {2}".format( + path, type(e).__name__, e + ) + logging.warning(errMsg) + return False + return True + + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["rmdir", path] + self.exec_command(cmd) + + def listdir(self, path): + """ + List all files and directories in a directory. + Args: + path (str): The path to the directory. + """ + command = ["ls", path] + output = self.exec_command(cmd=command, encoding=get_default_encoding()) + assert type(output) == str # noqa: E721 + result = output.splitlines() + assert type(result) == list # noqa: E721 + return result + + def path_exists(self, path): + command = ["test", "-e", path] + + exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return True + + if exit_status == 1: + return False + + errMsg = "Test operation returns an unknown result code: {0}. Path is [{1}].".format( + exit_status, + path) + + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exit_status, + message=errMsg, + error=error, + out=output + ) + + @property + def pathsep(self): + os_name = self.get_name() + if os_name == "posix": + pathsep = ":" + elif os_name == "nt": + pathsep = ";" + else: + raise Exception("Unsupported operating system: {}".format(os_name)) + return pathsep + + def mkdtemp(self, prefix=None): + """ + Creates a temporary directory in the remote server. + Args: + - prefix (str): The prefix of the temporary directory name. + """ + if prefix: + command = ["mktemp", "-d", "-t", prefix + "XXXXXX"] + else: + command = ["mktemp", "-d"] + + exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 + + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not create temporary directory.", + error=exec_error, + out=exec_output) + + temp_dir = exec_output.strip() + return temp_dir + + def mkstemp(self, prefix=None): + """ + Creates a temporary file in the remote server. + Args: + - prefix (str): The prefix of the temporary directory name. + """ + if prefix: + command = ["mktemp", "-t", prefix + "XXXXXX"] + else: + command = ["mktemp"] + + exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 + + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not create temporary file.", + error=exec_error, + out=exec_output) + + temp_file = exec_output.strip() + return temp_file + + def copytree(self, src, dst): + if not os.path.isabs(dst): + dst = os.path.join('~', dst) + if self.isdir(dst): + raise FileExistsError("Directory {} already exists.".format(dst)) + return self.exec_command("cp -r {} {}".format(src, dst)) + + # Work with files + def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=None): + if not encoding: + encoding = get_default_encoding() + mode = "wb" if binary else "w" + + with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + # For scp the port is specified by a "-P" option + scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] + + if not truncate: + scp_cmd = ['scp'] + scp_args + [f"{self.ssh_dest}:{filename}", tmp_file.name] + subprocess.run(scp_cmd, check=False) # The file might not exist yet + tmp_file.seek(0, os.SEEK_END) + + if isinstance(data, list): + data2 = [__class__._prepare_line_to_write(s, binary, encoding) for s in data] + tmp_file.writelines(data2) + else: + data2 = __class__._prepare_data_to_write(data, binary, encoding) + tmp_file.write(data2) + + tmp_file.flush() + scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.ssh_dest}:{filename}"] + subprocess.run(scp_cmd, check=True) + + remote_directory = os.path.dirname(filename) + mkdir_cmd = ['ssh'] + self.ssh_args + [self.ssh_dest, f"mkdir -p {remote_directory}"] + subprocess.run(mkdir_cmd, check=True) + + os.remove(tmp_file.name) + + @staticmethod + def _prepare_line_to_write(data, binary, encoding): + data = __class__._prepare_data_to_write(data, binary, encoding) + + if binary: + assert type(data) == bytes # noqa: E721 + return data.rstrip(b'\n') + b'\n' + + assert type(data) == str # noqa: E721 + return data.rstrip('\n') + '\n' + + @staticmethod + def _prepare_data_to_write(data, binary, encoding): + if isinstance(data, bytes): + return data if binary else data.decode(encoding) + + if isinstance(data, str): + return data if not binary else data.encode(encoding) + + raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) + + def touch(self, filename): + """ + Create a new file or update the access and modification times of an existing file on the remote server. + + Args: + filename (str): The name of the file to touch. + + This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. + """ + self.exec_command("touch {}".format(filename)) + + def read(self, filename, binary=False, encoding=None): + assert type(filename) == str # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + + if binary: + if encoding is not None: + raise InvalidOperationException("Enconding is not allowed for read binary operation") + + return self._read__binary(filename) + + # python behavior + assert (None or "abc") == "abc" + assert ("" or "abc") == "abc" + + return self._read__text_with_encoding(filename, encoding or get_default_encoding()) + + def _read__text_with_encoding(self, filename, encoding): + assert type(filename) == str # noqa: E721 + assert type(encoding) == str # noqa: E721 + content = self._read__binary(filename) + assert type(content) == bytes # noqa: E721 + buf0 = io.BytesIO(content) + buf1 = io.TextIOWrapper(buf0, encoding=encoding) + content_s = buf1.read() + assert type(content_s) == str # noqa: E721 + return content_s + + def _read__binary(self, filename): + assert type(filename) == str # noqa: E721 + cmd = ["cat", filename] + content = self.exec_command(cmd) + assert type(content) == bytes # noqa: E721 + return content + + def readlines(self, filename, num_lines=0, binary=False, encoding=None): + assert type(num_lines) == int # noqa: E721 + assert type(filename) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + + if num_lines > 0: + cmd = ["tail", "-n", str(num_lines), filename] + else: + cmd = ["cat", filename] + + if binary: + assert encoding is None + pass + elif encoding is None: + encoding = get_default_encoding() + assert type(encoding) == str # noqa: E721 + else: + assert type(encoding) == str # noqa: E721 + pass + + result = self.exec_command(cmd, encoding=encoding) + assert result is not None + + if binary: + assert type(result) == bytes # noqa: E721 + lines = result.splitlines() + else: + assert type(result) == str # noqa: E721 + lines = result.splitlines() + + assert type(lines) == list # noqa: E721 + return lines + + def read_binary(self, filename, offset): + assert type(filename) == str # noqa: E721 + assert type(offset) == int # noqa: E721 + + if offset < 0: + raise ValueError("Negative 'offset' is not supported.") + + cmd = ["tail", "-c", "+{}".format(offset + 1), filename] + r = self.exec_command(cmd) + assert type(r) == bytes # noqa: E721 + return r + + def isfile(self, remote_file): + stdout = self.exec_command("test -f {}; echo $?".format(remote_file)) + result = int(stdout.strip()) + return result == 0 + + def isdir(self, dirname): + cmd = "if [ -d {} ]; then echo True; else echo False; fi".format(dirname) + response = self.exec_command(cmd) + return response.strip() == b"True" + + def get_file_size(self, filename): + C_ERR_SRC = "RemoteOpertions::get_file_size" + + assert filename is not None + assert type(filename) == str # noqa: E721 + cmd = ["du", "-b", filename] + + s = self.exec_command(cmd, encoding=get_default_encoding()) + assert type(s) == str # noqa: E721 + + if len(s) == 0: + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned an empty string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#001", + filename + ) + ) + + i = 0 + + while i < len(s) and s[i].isdigit(): + assert s[i] >= '0' + assert s[i] <= '9' + i += 1 + + if i == 0: + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#002", + filename + ) + ) + + if i == len(s): + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#003", + filename + ) + ) + + if not s[i].isspace(): + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#004", + filename + ) + ) + + r = 0 + + for i2 in range(0, i): + ch = s[i2] + assert ch >= '0' + assert ch <= '9' + # Here is needed to check overflow or that it is a human-valid result? + r = (r * 10) + ord(ch) - ord('0') + + return r + + def remove_file(self, filename): + cmd = "rm {}".format(filename) + return self.exec_command(cmd) + + # Processes control + def kill(self, pid, signal): + # Kill the process + cmd = "kill -{} {}".format(signal, pid) + return self.exec_command(cmd) + + def get_pid(self): + # Get current process id + return int(self.exec_command("echo $$", encoding=get_default_encoding())) + + def get_process_children(self, pid): + assert type(pid) == int # noqa: E721 + command = ["ssh"] + self.ssh_args + [self.ssh_dest, "pgrep", "-P", str(pid)] + + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + if result.returncode == 0: + children = result.stdout.strip().splitlines() + return [PsUtilProcessProxy(self, int(child_pid.strip())) for child_pid in children] + + raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") + + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + cmd = ["nc", "-w", "5", "-z", "-v", "localhost", str(number)] + + exit_status, output, error = self.exec_command(cmd=cmd, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return __class__._is_port_free__process_0(error) + + if exit_status == 1: + return __class__._is_port_free__process_1(error) + + errMsg = "nc returns an unknown result code: {0}".format(exit_status) + + RaiseError.CommandExecutionError( + cmd=cmd, + exit_code=exit_status, + message=errMsg, + error=error, + out=output + ) + + def get_tempdir(self) -> str: + command = ["mktemp", "-u", "-d"] + + exec_exitcode, exec_output, exec_error = self.exec_command( + command, + verbose=True, + encoding=get_default_encoding(), + ignore_errors=True + ) + + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 + + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not detect a temporary directory.", + error=exec_error, + out=exec_output) + + temp_subdir = exec_output.strip() + assert type(temp_subdir) == str # noqa: E721 + temp_dir = os.path.dirname(temp_subdir) + assert type(temp_dir) == str # noqa: E721 + return temp_dir + + @staticmethod + def _is_port_free__process_0(error: str) -> bool: + assert type(error) == str # noqa: E721 + # + # Example of error text: + # "Connection to localhost (127.0.0.1) 1024 port [tcp/*] succeeded!\n" + # + # May be here is needed to check error message? + # + return False + + @staticmethod + def _is_port_free__process_1(error: str) -> bool: + assert type(error) == str # noqa: E721 + # + # Example of error text: + # "nc: connect to localhost (127.0.0.1) port 1024 (tcp) failed: Connection refused\n" + # + # May be here is needed to check error message? + # + return True + + @staticmethod + def _make_exec_env_list(exec_env: typing.Dict) -> typing.List[str]: + env: typing.Dict[str, str] = dict() + + # ---------------------------------- SYSTEM ENV + for envvar in os.environ.items(): + if __class__._does_put_envvar_into_exec_cmd(envvar[0]): + env[envvar[0]] = envvar[1] + + # ---------------------------------- EXEC (LOCAL) ENV + if exec_env is None: + pass + else: + for envvar in exec_env.items(): + assert type(envvar) == tuple # noqa: E721 + assert len(envvar) == 2 + assert type(envvar[0]) == str # noqa: E721 + env[envvar[0]] = envvar[1] + + # ---------------------------------- FINAL BUILD + result: typing.List[str] = list() + for envvar in env.items(): + assert type(envvar) == tuple # noqa: E721 + assert len(envvar) == 2 + assert type(envvar[0]) == str # noqa: E721 + + if envvar[1] is None: + result.append("unset " + envvar[0]) + else: + assert type(envvar[1]) == str # noqa: E721 + qvalue = __class__._quote_envvar(envvar[1]) + assert type(qvalue) == str # noqa: E721 + result.append(envvar[0] + "=" + qvalue) + continue + + return result + + sm_envs_for_exec_cmd = ["LANG", "LANGUAGE"] + + @staticmethod + def _does_put_envvar_into_exec_cmd(name: str) -> bool: + assert type(name) == str # noqa: E721 + name = name.upper() + if name.startswith("LC_"): + return True + if name in __class__.sm_envs_for_exec_cmd: + return True + return False + + @staticmethod + def _quote_envvar(value: str) -> str: + assert type(value) == str # noqa: E721 + result = "\"" + for ch in value: + if ch == "\"": + result += "\\\"" + elif ch == "\\": + result += "\\\\" + else: + result += ch + result += "\"" + return result + + +def normalize_error(error): + if isinstance(error, bytes): + return error.decode() + return error diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py new file mode 100644 index 00000000..824eadc6 --- /dev/null +++ b/testgres/plugins/__init__.py @@ -0,0 +1,8 @@ +from .pg_probackup2.pg_probackup2.gdb import GDBobj +from .pg_probackup2.pg_probackup2.app import ProbackupApp, ProbackupException +from .pg_probackup2.pg_probackup2.init_helpers import init_params +from .pg_probackup2.pg_probackup2.storage.fs_backup import FSTestBackupDir + +__all__ = [ + "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" +] diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md new file mode 100644 index 00000000..5139ab0f --- /dev/null +++ b/testgres/plugins/pg_probackup2/README.md @@ -0,0 +1,57 @@ +# testgres - pg_probackup2 + +Ccontrol and testing utility for [pg_probackup2](https://github.com/postgrespro/pg_probackup). Python 3.5+ is supported. + + +## Installation + +To install `testgres`, run: + +``` +pip install testgres-pg_probackup +``` + +We encourage you to use `virtualenv` for your testing environment. +The package requires testgres~=1.9.3. + +## Usage + +### Environment variables + +| Variable | Required | Default value | Description | +| - | - | - | - | +| PGPROBACKUP_TMP_DIR | No | tests/tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the current working directory. | +| PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX | No | Temporary test hierarchy | Prefix of the test backup directories. Must be an absolute path. Use this variable to store test backups in a location other than the temporary test hierarchy. | + +See [Testgres](https://github.com/postgrespro/testgres/tree/master#environment) on how to configure a custom Postgres installation using `PG_CONFIG` and `PG_BIN` environment variables. + +### Examples + +Here is an example of what you can do with `testgres-pg_probackup2`: + +```python +# You can see full script here plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +def test_full_backup(self): + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + self.assertIn(f"INFO: Backup {backup_id} is valid", out) +``` + +## Authors + +[Postgres Professional](https://postgrespro.ru/about) diff --git a/testgres/plugins/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py new file mode 100644 index 00000000..5166e9b8 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -0,0 +1,863 @@ +import contextlib +import importlib +import json +import logging +import os +import re +import subprocess +import threading +import time +import unittest + +import testgres + +from .storage.fs_backup import TestBackupDir, FSTestBackupDir +from .gdb import GDBobj +from .init_helpers import init_params + +warning = """ +Wrong splint in show_pb +Original Header:f +{header} +Original Body: +{body} +Splitted Header +{header_split} +Splitted Body +{body_split} +""" + + +class ProbackupException(Exception): + def __init__(self, message, cmd): + self.message = message + self.cmd = cmd + + def __str__(self): + return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + + +# Local backup control +fs_backup_class = FSTestBackupDir + + +class ProbackupApp: + + def __init__(self, test_class: unittest.TestCase, + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): + self.process = None + self.test_class = test_class + self.pg_node = pg_node + self.pb_log_path = pb_log_path + self.test_env = test_env + self.auto_compress_alg = auto_compress_alg + self.backup_dir = backup_dir + self.probackup_path = probackup_path or init_params.probackup_path + self.probackup_old_path = init_params.probackup_old_path + self.remote = init_params.remote + self.wal_tree_enabled = init_params.wal_tree_enabled + self.verbose = init_params.verbose + self.archive_compress = init_params.archive_compress + self.test_class.output = None + self.execution_time = None + + def form_daemon_process(self, cmdline, env): + def stream_output(stream: subprocess.PIPE) -> None: + try: + for line in iter(stream.readline, ''): + print(line) + self.test_class.output += line + finally: + stream.close() + + self.process = subprocess.Popen( + cmdline, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env + ) + logging.info(f"Process started in background with PID: {self.process.pid}") + + if self.process.stdout and self.process.stderr: + stdout_thread = threading.Thread(target=stream_output, args=(self.process.stdout,), daemon=True) + stderr_thread = threading.Thread(target=stream_output, args=(self.process.stderr,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + return self.process.pid + + def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, + skip_log_directory=False, expect_error=False, use_backup_dir=True, daemonize=False): + """ + Run pg_probackup + backup_dir: target directory for making backup + command: commandline options + expect_error: option for ignoring errors and getting error message as a result of running the function + gdb: when True it returns GDBObj(), when tuple('suspend', port) it runs probackup + in suspended gdb mode with attachable gdb port, for local debugging + """ + if isinstance(use_backup_dir, TestBackupDir): + command = [command[0], *use_backup_dir.pb_args, *command[1:]] + elif use_backup_dir: + command = [command[0], *self.backup_dir.pb_args, *command[1:]] + else: + command = [command[0], *self.backup_dir.pb_args[2:], *command[1:]] + + if not self.probackup_old_path and old_binary: + logging.error('PGPROBACKUPBIN_OLD is not set') + exit(1) + + if old_binary: + binary_path = self.probackup_old_path + else: + binary_path = self.probackup_path + + if not env: + env = self.test_env + + strcommand = ' '.join(str(p) for p in command) + if '--log-level-file' in strcommand and \ + '--log-directory' not in strcommand and \ + not skip_log_directory: + command += ['--log-directory=' + self.pb_log_path] + strcommand += ' ' + command[-1] + + if 'pglz' in strcommand and \ + ' -j' not in strcommand and '--thread' not in strcommand: + command += ['-j', '1'] + strcommand += ' -j 1' + + self.test_class.cmd = binary_path + ' ' + strcommand + if self.verbose: + print(self.test_class.cmd) + + cmdline = [binary_path, *command] + if gdb is True: + # general test flow for using GDBObj + return GDBobj(cmdline, self.test_class) + + try: + if type(gdb) is tuple and gdb[0] == 'suspend': + # special test flow for manually debug probackup + gdb_port = gdb[1] + cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline + logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + + start_time = time.time() + if daemonize: + return self.form_daemon_process(cmdline, env) + else: + self.test_class.output = subprocess.check_output( + cmdline, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8', errors='replace') + end_time = time.time() + self.execution_time = end_time - start_time + + if command[0] == 'backup' and return_id: + result = self.get_backup_id() + else: + result = self.test_class.output + if expect_error is True: + assert False, f"Exception was expected, but run finished successful with result: `{result}`\n" \ + f"CMD: {self.test_class.cmd}" + elif expect_error: + assert False, f"Exception was expected {expect_error}, but run finished successful with result: `{result}`\n" \ + f"CMD: {self.test_class.cmd}" + return result + except subprocess.CalledProcessError as e: + self.test_class.output = e.output.decode('utf-8').replace("\r", "") + if expect_error: + return self.test_class.output + else: + raise ProbackupException(self.test_class.output, self.test_class.cmd) + + def get_backup_id(self): + if init_params.major_version > 2: + pattern = re.compile(r"Backup (.*) completed successfully.") + for line in self.test_class.output.splitlines(): + match = pattern.search(line) + if match: + return match.group(1) + else: + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + return None + + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): + if options is None: + options = [] + return self.run(['init'] + options, + old_binary=old_binary, + skip_log_directory=skip_log_directory, + expect_error=expect_error, + use_backup_dir=use_backup_dir + ) + + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False): + if options is None: + options = [] + + if not datname: + datname = 'postgres' + + cmd = [ + 'add-instance', + '--instance={0}'.format(instance), + '-D', node.data_dir, + '--pgport', '%i' % node.port, + '--pgdatabase', datname + ] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + if self.wal_tree_enabled: + options = options + ['--wal-tree'] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def set_config(self, instance, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'set-config', + '--instance={0}'.format(instance), + ] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def set_backup(self, instance, backup_id=False, + old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'set-backup', + ] + + if instance: + cmd = cmd + ['--instance={0}'.format(instance)] + + if backup_id: + cmd = cmd + ['-i', backup_id] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def del_instance(self, instance, options=None, old_binary=False, expect_error=False): + if options is None: + options = [] + cmd = ['del-instance', '--instance={0}'.format(instance)] + options + return self.run(cmd, + old_binary=old_binary, + expect_error=expect_error) + + def backup_node( + self, instance, node, data_dir=False, + backup_type='full', datname=False, options=None, + gdb=False, + old_binary=False, return_id=True, no_remote=False, + env=None, + expect_error=False, + sync=False + ): + if options is None: + options = [] + if not node and not data_dir: + logging.error('You must provide ether node or data_dir for backup') + exit(1) + + if not datname: + datname = 'postgres' + + cmd_list = [ + 'backup', + '--instance={0}'.format(instance), + # "-D", pgdata, + '-p', '%i' % node.port, + '-d', datname + ] + + if data_dir: + cmd_list += ['-D', self._node_dir(data_dir)] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary and not no_remote: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + if self.auto_compress_alg and '--compress' in options and \ + self.archive_compress and self.archive_compress != 'zlib': + options = [o if o != '--compress' else f'--compress-algorithm={self.archive_compress}' + for o in options] + + if backup_type: + cmd_list += ['-b', backup_type] + + if not (old_binary or sync): + cmd_list += ['--no-sync'] + + return self.run(cmd_list + options, gdb, old_binary, return_id, env=env, + expect_error=expect_error) + + def backup_replica_node(self, instance, node, data_dir=False, *, + master, backup_type='full', datname=False, + options=None, env=None): + """ + Try to reliably run backup on replica by switching wal at master + at the moment pg_probackup is waiting for archived wal segment + """ + if options is None: + options = [] + assert '--stream' not in options or backup_type == 'page', \ + "backup_replica_node should be used with one of archive-mode or " \ + "page-stream mode" + + options = options.copy() + if not any('--log-level-file' in x for x in options): + options.append('--log-level-file=INFO') + + gdb = self.backup_node( + instance, node, data_dir, + backup_type=backup_type, + datname=datname, + options=options, + env=env, + gdb=True) + gdb.set_breakpoint('wait_wal_lsn') + # we need to break on wait_wal_lsn in pg_stop_backup + gdb.run_until_break() + if backup_type == 'page': + self.switch_wal_segment(master) + if '--stream' not in options: + gdb.continue_execution_until_break() + self.switch_wal_segment(master) + gdb.continue_execution_until_exit() + + output = self.read_pb_log() + self.unlink_pg_log() + parsed_output = re.compile(r'Backup \S+ completed').search(output) + assert parsed_output, f"Expected: `Backup 'backup_id' completed`, but found `{output}`" + backup_id = parsed_output[0].split(' ')[1] + return (backup_id, output) + + def checkdb_node( + self, use_backup_dir=False, instance=False, data_dir=False, + options=None, gdb=False, old_binary=False, + skip_log_directory=False, + expect_error=False + ): + if options is None: + options = [] + cmd_list = ["checkdb"] + + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if data_dir: + cmd_list += ["-D", self._node_dir(data_dir)] + + return self.run(cmd_list + options, gdb, old_binary, + skip_log_directory=skip_log_directory, expect_error=expect_error, + use_backup_dir=use_backup_dir) + + def merge_backup( + self, instance, backup_id, + gdb=False, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'merge', + '--instance={0}'.format(instance), + '-i', backup_id + ] + + return self.run(cmd_list + options, gdb, old_binary, expect_error=expect_error) + + def restore_node( + self, instance, node=None, restore_dir=None, + backup_id=None, old_binary=False, options=None, + gdb=False, + expect_error=False, + sync=False + ): + if options is None: + options = [] + if node: + if isinstance(node, str): + data_dir = node + else: + data_dir = node.data_dir + elif restore_dir: + data_dir = self._node_dir(restore_dir) + else: + raise ValueError("You must provide ether node or base_dir for backup") + + cmd_list = [ + 'restore', + '-D', data_dir, + '--instance={0}'.format(instance) + ] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + if backup_id: + cmd_list += ['-i', backup_id] + + if not (old_binary or sync): + cmd_list += ['--no-sync'] + + return self.run(cmd_list + options, gdb=gdb, old_binary=old_binary, expect_error=expect_error) + + def catchup_node( + self, + backup_mode, source_pgdata, destination_node, + options=None, + remote_host='localhost', + remote_port=None, + expect_error=False, + gdb=False + ): + + if options is None: + options = [] + cmd_list = [ + 'catchup', + '--backup-mode={0}'.format(backup_mode), + '--source-pgdata={0}'.format(source_pgdata), + '--destination-pgdata={0}'.format(destination_node.data_dir) + ] + if self.remote: + cmd_list += ['--remote-proto=ssh', f'--remote-host={remote_host}'] + if remote_port: + cmd_list.append(f'--remote-port={remote_port}') + if self.verbose: + cmd_list += [ + '--log-level-file=VERBOSE', + '--log-directory={0}'.format(destination_node.logs_dir) + ] + + return self.run(cmd_list + options, gdb=gdb, expect_error=expect_error, use_backup_dir=False) + + def show( + self, instance=None, backup_id=None, + options=None, as_text=False, as_json=True, old_binary=False, + env=None, + expect_error=False, + gdb=False + ): + + if options is None: + options = [] + backup_list = [] + specific_record = {} + cmd_list = [ + 'show', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + + if backup_id: + cmd_list += ['-i', backup_id] + + # AHTUNG, WARNING will break json parsing + if as_json: + cmd_list += ['--format=json', '--log-level-console=error'] + + if as_text: + # You should print it when calling as_text=true + return self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error, gdb=gdb) + + # get show result as list of lines + if as_json: + text_json = str(self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error, gdb=gdb)) + try: + if expect_error: + return text_json + data = json.loads(text_json) + except ValueError: + assert False, f"Couldn't parse {text_json} as json. " \ + f"Check that you don't have additional messages inside the log or use 'as_text=True'" + + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: + continue + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) + + if backup_id is not None: + assert False, "Failed to find backup with ID: {0}".format(backup_id) + + return backup_list + else: + show_splitted = self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(' +', header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(' +', backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + logging.error(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(' = ')[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + + if not specific_record: + assert False, "Failed to find backup with ID: {0}".format(backup_id) + + return specific_record + + def show_archive( + self, instance=None, options=None, + as_text=False, as_json=True, old_binary=False, + tli=0, + expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'show', + '--archive', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + + # AHTUNG, WARNING will break json parsing + if as_json: + cmd_list += ['--format=json', '--log-level-console=error'] + + if as_text: + # You should print it when calling as_text=true + return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + + if as_json: + if as_text: + data = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + else: + data = json.loads(self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error)) + + if instance: + instance_timelines = None + for instance_name in data: + if instance_name['instance'] == instance: + instance_timelines = instance_name['timelines'] + break + + if tli > 0: + for timeline in instance_timelines: + if timeline['tli'] == tli: + return timeline + + return {} + + if instance_timelines: + return instance_timelines + + return data + else: + show_splitted = self.run(cmd_list + options, old_binary=old_binary, + expect_error=expect_error).splitlines() + logging.error(show_splitted) + exit(1) + + def validate( + self, instance=None, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'validate', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + + def delete( + self, instance, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'delete', + ] + + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + + def delete_expired( + self, instance, options=None, old_binary=False, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'delete', + '--instance={0}'.format(instance) + ] + return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + + def show_config(self, instance, old_binary=False, expect_error=False, gdb=False): + out_dict = {} + cmd_list = [ + 'show-config', + '--instance={0}'.format(instance) + ] + + res = self.run(cmd_list, old_binary=old_binary, expect_error=expect_error, gdb=gdb).splitlines() + for line in res: + if not line.startswith('#'): + name, var = line.partition(' = ')[::2] + out_dict[name] = var + return out_dict + + def run_binary(self, command, asynchronous=False, env=None): + + if not env: + env = self.test_env + + if self.verbose: + print([' '.join(map(str, command))]) + try: + if asynchronous: + return subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env + ) + else: + self.test_class.output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8') + return self.test_class.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode('utf-8'), command) + + def _node_dir(self, base_dir): + return os.path.join(self.pg_node.test_path, base_dir) + + def set_archiving( + self, instance, node, replica=False, + overwrite=False, compress=True, old_binary=False, + log_level=False, archive_timeout=False, + custom_archive_command=None): + + # parse postgresql.auto.conf + options = {} + if replica: + options['archive_mode'] = 'always' + options['hot_standby'] = 'on' + else: + options['archive_mode'] = 'on' + + if custom_archive_command is None: + archive_command = " ".join([f'"{init_params.probackup_path}"', + 'archive-push', *self.backup_dir.pb_args]) + if os.name == 'posix': + # Dash produces a core dump when it gets a SIGQUIT from its + # child process so replace the shell with pg_probackup + archive_command = 'exec ' + archive_command + elif os.name == "nt": + archive_command = archive_command.replace("\\", "\\\\") + archive_command += f' --instance={instance}' + + # don`t forget to kill old_binary after remote ssh release + if init_params.remote and not old_binary: + archive_command += ' --remote-proto=ssh --remote-host=localhost' + + if init_params.archive_compress and compress: + archive_command += ' --compress-algorithm=' + init_params.archive_compress + + if overwrite: + archive_command += ' --overwrite' + + if init_params.major_version > 2: + archive_command += ' --log-level-console=trace' + else: + archive_command += ' --log-level-console=VERBOSE' + archive_command += ' -j 5' + archive_command += ' --batch-size 10' + + archive_command += ' --no-sync' + + if archive_timeout: + archive_command += f' --archive-timeout={archive_timeout}' + + if os.name == 'posix': + archive_command += ' --wal-file-path=%p --wal-file-name=%f' + + elif os.name == 'nt': + archive_command += ' --wal-file-path="%p" --wal-file-name="%f"' + + if log_level: + archive_command += f' --log-level-console={log_level}' + else: # custom_archive_command is not None + archive_command = custom_archive_command + options['archive_command'] = archive_command + + node.set_auto_conf(options) + + def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): + """ + Execute pg_switch_wal() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + lsn = con.execute('select pg_switch_wal()')[0][0] + else: + lsn = node.execute('select pg_switch_wal()')[0][0] + + if sleep_seconds > 0: + time.sleep(sleep_seconds) + return lsn + + @contextlib.contextmanager + def switch_wal_after(self, node, seconds, and_tx=True): + tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) + tm.start() + try: + yield + finally: + tm.cancel() + tm.join() + + def read_pb_log(self): + with open(os.path.join(self.pb_log_path, 'pg_probackup.log')) as fl: + return fl.read() + + def unlink_pg_log(self): + os.unlink(os.path.join(self.pb_log_path, 'pg_probackup.log')) + + def load_backup_class(fs_type): + fs_type = os.environ.get('PROBACKUP_FS_TYPE') + implementation = f"{__package__}.fs_backup.FSTestBackupDir" + if fs_type: + implementation = fs_type + + logging.info("Using ", implementation) + module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) + + module = importlib.import_module(module_name) + + return getattr(module, class_name) + + def archive_push(self, instance, node, wal_file_name, wal_file_path=None, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'archive-push', + '--instance={0}'.format(instance), + '--wal-file-name={0}'.format(wal_file_name), + ] + if wal_file_path is None: + cmd = cmd + ['--wal-file-path={0}'.format(os.path.join(node.data_dir, 'pg_wal'))] + else: + cmd = cmd + ['--wal-file-path={0}'.format(wal_file_path)] + return self.run(cmd + options, expect_error=expect_error) + + def archive_get(self, instance, wal_file_name, wal_file_path, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'archive-get', + '--instance={0}'.format(instance), + '--wal-file-name={0}'.format(wal_file_name), + '--wal-file-path={0}'.format(wal_file_path), + ] + return self.run(cmd + options, expect_error=expect_error) + + def maintain( + self, instance=None, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'maintain', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + + def build_backup_dir(self, backup='backup'): + return fs_backup_class(rel_path=self.rel_path, backup=backup) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py new file mode 100644 index 00000000..2424c04d --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -0,0 +1,349 @@ +import functools +import os +import re +import subprocess +import sys +import unittest +from time import sleep + + +class GdbException(Exception): + def __init__(self, message="False"): + self.message = message + + def __str__(self): + return '\n ERROR: {0}\n'.format(repr(self.message)) + + +class GDBobj: + _gdb_enabled = False + _gdb_ok = False + _gdb_ptrace_ok = False + + def __init__(self, cmd, env, attach=False): + self.verbose = env.verbose + self.output = '' + self._did_quit = False + self.has_breakpoint = False + + # Check gdb flag is set up + if not hasattr(env, "_gdb_decorated") or not env._gdb_decorated: + raise GdbException("Test should be decorated with @needs_gdb") + if not self._gdb_enabled: + raise GdbException("No `PGPROBACKUP_GDB=on` is set.") + if not self._gdb_ok: + if not self._gdb_ptrace_ok: + raise GdbException("set /proc/sys/kernel/yama/ptrace_scope to 0" + " to run GDB tests") + raise GdbException("No gdb usage possible.") + + # Check gdb presence + try: + gdb_version, _ = subprocess.Popen( + ['gdb', '--version'], + stdout=subprocess.PIPE + ).communicate() + except OSError: + raise GdbException("Couldn't find gdb on the path") + + self.base_cmd = [ + 'gdb', + '--interpreter', + 'mi2', + ] + + if attach: + self.cmd = self.base_cmd + ['--pid'] + cmd + else: + self.cmd = self.base_cmd + ['--args'] + cmd + + # Get version + gdb_version_number = re.search( + br"^GNU gdb [^\d]*(\d+)\.(\d)", + gdb_version) + self.major_version = int(gdb_version_number.group(1)) + self.minor_version = int(gdb_version_number.group(2)) + + if self.verbose: + print([' '.join(map(str, self.cmd))]) + + self.proc = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + text=True, + errors='replace', + ) + self.gdb_pid = self.proc.pid + + while True: + line = self.get_line() + + if 'No such process' in line: + raise GdbException(line) + + if not line.startswith('(gdb)'): + pass + else: + break + + def __del__(self): + if not self._did_quit and hasattr(self, "proc"): + try: + self.quit() + except subprocess.TimeoutExpired: + self.kill() + + def get_line(self): + line = self.proc.stdout.readline() + self.output += line + return line + + def kill(self): + self._did_quit = True + self.proc.kill() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() + + def terminate_subprocess(self): + self._execute('kill') + + def set_breakpoint(self, location): + + result = self._execute('break ' + location) + self.has_breakpoint = True + for line in result: + if line.startswith('~"Breakpoint'): + return + + elif line.startswith('=breakpoint-created'): + return + + elif line.startswith('^error'): # or line.startswith('(gdb)'): + break + + elif line.startswith('&"break'): + pass + + elif line.startswith('&"Function'): + raise GdbException(line) + + elif line.startswith('&"No line'): + raise GdbException(line) + + elif line.startswith('~"Make breakpoint pending on future shared'): + raise GdbException(line) + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + + def remove_all_breakpoints(self): + if not self.has_breakpoint: + return + + result = self._execute('delete') + self.has_breakpoint = False + for line in result: + + if line.startswith('^done'): + return + + raise GdbException( + 'Failed to remove breakpoints.\n Output:\n {0}'.format(result) + ) + + def run_until_break(self): + result = self._execute('run', False) + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + raise GdbException( + 'Failed to run until breakpoint.\n' + ) + + def continue_execution_until_running(self): + result = self._execute('continue') + + for line in result: + if line.startswith('*running') or line.startswith('^running'): + return + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if line.startswith('*stopped,reason="exited-normally"'): + continue + + raise GdbException( + 'Failed to continue execution until running.\n' + ) + + def signal(self, sig): + if 'KILL' in sig: + self.remove_all_breakpoints() + self._execute(f'signal {sig}') + + def continue_execution_until_exit(self): + self.remove_all_breakpoints() + result = self._execute('continue', False) + + for line in result: + if line.startswith('*running'): + continue + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if line.startswith('*stopped,reason="exited') or line == '*stopped\n': + self.quit() + return + + raise GdbException( + 'Failed to continue execution until exit.\n' + ) + + def continue_execution_until_error(self): + self.remove_all_breakpoints() + result = self._execute('continue', False) + + for line in result: + if line.startswith('^error'): + return + if line.startswith('*stopped,reason="exited'): + return + if line.startswith( + '*stopped,reason="signal-received",signal-name="SIGABRT"'): + return + + raise GdbException( + 'Failed to continue execution until error.\n') + + def continue_execution_until_break(self, ignore_count=0): + if ignore_count > 0: + result = self._execute( + 'continue ' + str(ignore_count), + False + ) + else: + result = self._execute('continue', False) + + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + if line.startswith('*stopped,reason="exited-normally"'): + break + + raise GdbException( + 'Failed to continue execution until break.\n') + + def show_backtrace(self): + return self._execute("backtrace", running=False) + + def stopped_in_breakpoint(self): + while True: + line = self.get_line() + if self.verbose: + print(line) + if line.startswith('*stopped,reason="breakpoint-hit"'): + return True + + def detach(self): + if not self._did_quit: + self._execute('detach') + + def quit(self): + if not self._did_quit: + self._did_quit = True + self.proc.terminate() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() + + # use for breakpoint, run, continue + def _execute(self, cmd, running=True): + output = [] + self.proc.stdin.flush() + self.proc.stdin.write(cmd + '\n') + self.proc.stdin.flush() + sleep(1) + + # look for command we just send + while True: + line = self.get_line() + if self.verbose: + print(repr(line)) + + if cmd not in line: + continue + else: + break + + while True: + line = self.get_line() + output += [line] + if self.verbose: + print(repr(line)) + if line.startswith('^done') or line.startswith('*stopped'): + break + if line.startswith('^error'): + break + if running and (line.startswith('*running') or line.startswith('^running')): + # if running and line.startswith('*running'): + break + return output + + +def _set_gdb(self): + test_env = os.environ.copy() + self._gdb_enabled = test_env.get('PGPROBACKUP_GDB') == 'ON' + self._gdb_ok = self._gdb_enabled + if not self._gdb_enabled or sys.platform != 'linux': + return + try: + with open('/proc/sys/kernel/yama/ptrace_scope') as f: + ptrace = f.read() + except FileNotFoundError: + self._gdb_ptrace_ok = True + return + self._gdb_ptrace_ok = int(ptrace) == 0 + self._gdb_ok = self._gdb_ok and self._gdb_ptrace_ok + + +def _check_gdb_flag_or_skip_test(): + if not GDBobj._gdb_enabled: + return ("skip", + "Specify PGPROBACKUP_GDB and build without " + "optimizations for run this test" + ) + if GDBobj._gdb_ok: + return None + if not GDBobj._gdb_ptrace_ok: + return ("fail", "set /proc/sys/kernel/yama/ptrace_scope to 0" + " to run GDB tests") + else: + return ("fail", "use of gdb is not possible") + + +def needs_gdb(func): + check = _check_gdb_flag_or_skip_test() + if not check: + @functools.wraps(func) + def ok_wrapped(self): + self._gdb_decorated = True + func(self) + + return ok_wrapped + reason = check[1] + if check[0] == "skip": + return unittest.skip(reason)(func) + elif check[0] == "fail": + @functools.wraps(func) + def fail_wrapper(self): + self.fail(reason) + + return fail_wrapper + else: + raise "Wrong action {0}".format(check) + + +_set_gdb(GDBobj) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py new file mode 100644 index 00000000..c4570a39 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -0,0 +1,226 @@ +import logging +from functools import reduce +import getpass +import os +import re +import shutil +import subprocess +import sys +import testgres + +try: + import lz4.frame # noqa: F401 + + HAVE_LZ4 = True +except ImportError as e: + HAVE_LZ4 = False + LZ4_error = e + +try: + import zstd # noqa: F401 + + HAVE_ZSTD = True +except ImportError as e: + HAVE_ZSTD = False + ZSTD_error = e + +delete_logs = os.getenv('KEEP_LOGS') not in ['1', 'y', 'Y'] + +try: + testgres.configure_testgres( + cache_initdb=False, + cached_initdb_dir=False, + node_cleanup_full=delete_logs) +except Exception as e: + logging.warning("Can't configure testgres: {0}".format(e)) + + +class Init(object): + def __init__(self): + if '-v' in sys.argv or '--verbose' in sys.argv: + self.verbose = True + else: + self.verbose = False + + self._pg_config = testgres.get_pg_config() + self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' + self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' + self.is_pgpro = 'PGPRO_EDITION' in self._pg_config + self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] + self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] + version = self._pg_config['VERSION'].rstrip('develalphabetapre') + parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] + parts[0] = re.match(r'\d+', parts[0]).group() + self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + + os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale + test_env = os.environ.copy() + envs_list = [ + 'LANGUAGE', + 'LC_ALL', + 'PGCONNECT_TIMEOUT', + 'PGDATA', + 'PGDATABASE', + 'PGHOSTADDR', + 'PGREQUIRESSL', + 'PGSERVICE', + 'PGSSLMODE', + 'PGUSER', + 'PGPORT', + 'PGHOST' + ] + + for e in envs_list: + test_env.pop(e, None) + + test_env['LC_MESSAGES'] = 'C' + test_env['LC_TIME'] = 'C' + self._test_env = test_env + + # Get the directory from which the script was executed + self.source_path = os.getcwd() + tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') + if tmp_path and os.path.isabs(tmp_path): + self.tmp_path = tmp_path + else: + self.tmp_path = os.path.abspath( + os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) + ) + + os.makedirs(self.tmp_path, exist_ok=True) + + self.username = getpass.getuser() + + self.probackup_path = None + if 'PGPROBACKUPBIN' in test_env: + if shutil.which(test_env["PGPROBACKUPBIN"]): + self.probackup_path = test_env["PGPROBACKUPBIN"] + else: + if self.verbose: + print('PGPROBACKUPBIN is not an executable file') + + if not self.probackup_path: + probackup_path_tmp = os.path.join( + testgres.get_pg_config()['BINDIR'], 'pg_probackup') + + if os.path.isfile(probackup_path_tmp): + if not os.access(probackup_path_tmp, os.X_OK): + logging.warning('{0} is not an executable file'.format( + probackup_path_tmp)) + else: + self.probackup_path = probackup_path_tmp + + if not self.probackup_path: + probackup_path_tmp = self.source_path + + if os.path.isfile(probackup_path_tmp): + if not os.access(probackup_path_tmp, os.X_OK): + logging.warning('{0} is not an executable file'.format( + probackup_path_tmp)) + else: + self.probackup_path = probackup_path_tmp + + if not self.probackup_path: + raise Exception('pg_probackup binary is not found') + + if os.name == 'posix': + self.EXTERNAL_DIRECTORY_DELIMITER = ':' + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ':' + os.environ['PATH'] + + elif os.name == 'nt': + self.EXTERNAL_DIRECTORY_DELIMITER = ';' + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ';' + os.environ['PATH'] + + self.probackup_old_path = None + if 'PGPROBACKUPBIN_OLD' in test_env: + if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): + self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] + else: + if self.verbose: + print('PGPROBACKUPBIN_OLD is not an executable file') + + self.probackup_version = None + self.old_probackup_version = None + + probackup_version_output = subprocess.check_output( + [self.probackup_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + match = re.search(r"\d+\.\d+\.\d+", + probackup_version_output) + self.probackup_version = match.group(0) if match else None + match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) + compressions = match.group(1) if match else None + if compressions: + self.probackup_compressions = {s.strip() for s in compressions.split(',')} + else: + self.probackup_compressions = [] + + if self.probackup_old_path: + old_probackup_version_output = subprocess.check_output( + [self.probackup_old_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + match = re.search(r"\d+\.\d+\.\d+", + old_probackup_version_output) + self.old_probackup_version = match.group(0) if match else None + + self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' + self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' + + self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() + if self.bckp_source not in ('base', 'direct', 'pro'): + raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") + + self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' + env_compress = test_env.get('ARCHIVE_COMPRESSION', None) + if env_compress: + env_compress = env_compress.lower() + if env_compress in ('on', 'zlib'): + self.compress_suffix = '.gz' + self.archive_compress = 'zlib' + elif env_compress == 'lz4': + if not HAVE_LZ4: + raise LZ4_error + if 'lz4' not in self.probackup_compressions: + raise Exception("pg_probackup is not compiled with lz4 support") + self.compress_suffix = '.lz4' + self.archive_compress = 'lz4' + elif env_compress == 'zstd': + if not HAVE_ZSTD: + raise ZSTD_error + if 'zstd' not in self.probackup_compressions: + raise Exception("pg_probackup is not compiled with zstd support") + self.compress_suffix = '.zst' + self.archive_compress = 'zstd' + else: + self.compress_suffix = '' + self.archive_compress = False + + cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) + if cfs_compress: + self.cfs_compress = cfs_compress.lower() + else: + self.cfs_compress = self.archive_compress + + os.environ["PGAPPNAME"] = "pg_probackup" + self.delete_logs = delete_logs + + if self.probackup_version.split('.')[0].isdigit(): + self.major_version = int(self.probackup_version.split('.')[0]) + else: + raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) + + def test_env(self): + return self._test_env.copy() + + +try: + init_params = Init() +except Exception as e: + logging.error(str(e)) + logging.warning("testgres.plugins.probackup2.init_params is set to None.") + init_params = None diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py new file mode 100644 index 00000000..6c9d1463 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py @@ -0,0 +1,104 @@ +""" +Utilities for accessing pg_probackup backup data on file system. +""" +import os +import shutil + +from ..init_helpers import init_params + + +class TestBackupDir: + + def list_instance_backups(self, instance): + raise NotImplementedError() + + def list_files(self, sub_dir, recursive=False): + raise NotImplementedError() + + def list_dirs(self, sub_dir): + raise NotImplementedError() + + def read_file(self, sub_path, *, text=True): + raise NotImplementedError() + + def write_file(self, sub_path, data, *, text=True): + raise NotImplementedError() + + def cleanup(self): + raise NotImplementedError() + + def remove_file(self, sub_path): + raise NotImplementedError() + + def remove_dir(self, sub_path): + raise NotImplementedError() + + def exists(self, sub_path): + raise NotImplementedError() + + +class FSTestBackupDir(TestBackupDir): + is_file_based = True + + """ Backup directory. Usually created by running pg_probackup init -B """ + + def __init__(self, *, rel_path, backup): + backup_prefix = os.environ.get('PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX') + if backup_prefix and not os.path.isabs(backup_prefix): + raise Exception(f"PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX must be an absolute path, current value: {backup_prefix}") + self.path = os.path.join(backup_prefix or init_params.tmp_path, rel_path, backup) + self.pb_args = ('-B', self.path) + + def list_instance_backups(self, instance): + full_path = os.path.join(self.path, 'backups', instance) + return sorted((x for x in os.listdir(full_path) + if os.path.isfile(os.path.join(full_path, x, 'backup.control')))) + + def list_files(self, sub_dir, recursive=False): + full_path = os.path.join(self.path, sub_dir) + if not recursive: + return [f for f in os.listdir(full_path) + if os.path.isfile(os.path.join(full_path, f))] + files = [] + for rootdir, dirs, files_in_dir in os.walk(full_path): + rootdir = rootdir[len(self.path) + 1:] + files.extend(os.path.join(rootdir, file) for file in files_in_dir) + return files + + def list_dirs(self, sub_dir): + full_path = os.path.join(self.path, sub_dir) + return [f for f in os.listdir(full_path) + if os.path.isdir(os.path.join(full_path, f))] + + def read_file(self, sub_path, *, text=True): + full_path = os.path.join(self.path, sub_path) + with open(full_path, 'r' if text else 'rb') as fin: + return fin.read() + + def write_file(self, sub_path, data, *, text=True): + full_path = os.path.join(self.path, sub_path) + with open(full_path, 'w' if text else 'wb') as fout: + fout.write(data) + + def cleanup(self): + shutil.rmtree(self.path, ignore_errors=True) + + def remove_file(self, sub_path): + os.remove(os.path.join(self.path, sub_path)) + + def remove_dir(self, sub_path): + full_path = os.path.join(self.path, sub_path) + shutil.rmtree(full_path, ignore_errors=True) + + def exists(self, sub_path): + full_path = os.path.join(self.path, sub_path) + return os.path.exists(full_path) + + def __str__(self): + return self.path + + def __repr__(self): + return "FSTestBackupDir" + str(self.path) + + def __fspath__(self): + return self.path diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py new file mode 100644 index 00000000..2540ddb0 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import os +import shutil +import pytest + +import testgres +from ...pg_probackup2.app import ProbackupApp +from ...pg_probackup2.init_helpers import Init, init_params +from ..storage.fs_backup import FSTestBackupDir + + +class ProbackupTest: + pg_node: testgres.NodeApp + + @staticmethod + def probackup_is_available() -> bool: + p = os.environ.get("PGPROBACKUPBIN") + + if p is None: + return False + + if not os.path.exists(p): + return False + + return True + + @pytest.fixture(autouse=True, scope="function") + def implicit_fixture(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + self.helper__setUp(request) + yield + self.helper__tearDown() + + def helper__setUp(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + self.helper__setup_test_environment(request) + self.helper__setup_test_paths() + self.helper__setup_backup_dir() + self.helper__setup_probackup() + + def helper__setup_test_environment(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + self.output = None + self.cmd = None + self.nodes_to_cleanup = [] + self.module_name, self.fname = request.node.cls.__name__, request.node.name + self.test_env = Init().test_env() + + def helper__setup_test_paths(self): + self.rel_path = os.path.join(self.module_name, self.fname) + self.test_path = os.path.join(init_params.tmp_path, self.rel_path) + os.makedirs(self.test_path, exist_ok=True) + self.pb_log_path = os.path.join(self.test_path, "pb_log") + + def helper__setup_backup_dir(self): + self.backup_dir = self.helper__build_backup_dir('backup') + self.backup_dir.cleanup() + + def helper__setup_probackup(self): + self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) + self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, + auto_compress_alg='zlib', backup_dir=self.backup_dir) + + def helper__tearDown(self): + if os.path.exists(self.test_path): + shutil.rmtree(self.test_path) + + def helper__build_backup_dir(self, backup='backup'): + return FSTestBackupDir(rel_path=self.rel_path, backup=backup) + + +@pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") +class TestBasic(ProbackupTest): + def test_full_backup(self): + assert self.pg_node is not None + assert type(self.pg_node) == testgres.NodeApp # noqa: E721 + assert self.pb is not None + assert type(self.pb) == ProbackupApp # noqa: E721 + + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + + with node: + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + assert f"INFO: Backup {backup_id} is valid" in out diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py new file mode 100644 index 00000000..7a3212e4 --- /dev/null +++ b/testgres/plugins/pg_probackup2/setup.py @@ -0,0 +1,18 @@ +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + version='0.1.0', + name='testgres_pg_probackup2', + packages=['pg_probackup2', 'pg_probackup2.storage'], + description='Plugin for testgres that manages pg_probackup2', + url='https://github.com/postgrespro/testgres', + long_description_content_type='text/markdown', + license='PostgreSQL', + author='Postgres Professional', + author_email='testgres@postgrespro.ru', + keywords=['pg_probackup', 'testing', 'testgres'], + install_requires=['testgres>=1.9.2'] +) diff --git a/testgres/port_manager.py b/testgres/port_manager.py new file mode 100644 index 00000000..1ae696c8 --- /dev/null +++ b/testgres/port_manager.py @@ -0,0 +1,10 @@ +class PortManager: + def __init__(self): + super().__init__() + + def reserve_port(self) -> int: + raise NotImplementedError("PortManager::reserve_port is not implemented.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + raise NotImplementedError("PortManager::release_port is not implemented.") diff --git a/testgres/pubsub.py b/testgres/pubsub.py new file mode 100644 index 00000000..1be673bb --- /dev/null +++ b/testgres/pubsub.py @@ -0,0 +1,217 @@ +# coding: utf-8 +""" +Unlike physical replication the logical replication allows users replicate only +specified databases and tables. It uses publish-subscribe model with possibly +multiple publishers and multiple subscribers. When initializing publisher's +node ``allow_logical=True`` should be passed to the :meth:`.PostgresNode.init()` +method to enable PostgreSQL to write extra information to the WAL needed by +logical replication. + +To replicate table ``X`` from node A to node B the same table structure should +be defined on the subscriber's node as logical replication don't replicate DDL. +After that :meth:`~.PostgresNode.publish()` and :meth:`~.PostgresNode.subscribe()` +methods may be used to setup replication. Example: + +>>> from testgres import get_new_node +>>> with get_new_node() as nodeA, get_new_node() as nodeB: +... nodeA.init(allow_logical=True).start() +... nodeB.init().start() +... +... # create same table both on publisher and subscriber +... create_table = 'create table test (a int, b int)' +... nodeA.safe_psql(create_table) +... nodeB.safe_psql(create_table) +... +... # create publication +... pub = nodeA.publish('mypub') +... # create subscription +... sub = nodeB.subscribe(pub, 'mysub') +... +... # insert some data to the publisher's node +... nodeA.execute('insert into test values (1, 1), (2, 2)') +... +... # wait until changes apply on subscriber and check them +... sub.catchup() +... +... # read the data from subscriber's node +... nodeB.execute('select * from test') +PostgresNode(name='...', port=..., base_dir='...') +PostgresNode(name='...', port=..., base_dir='...') +'' +'' +[(1, 1), (2, 2)] +""" + +from six import raise_from + +from .consts import LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS +from .defaults import default_dbname, default_username +from .exceptions import CatchUpException +from .utils import options_string + + +class Publication(object): + def __init__(self, name, node, tables=None, dbname=None, username=None): + """ + Constructor. Use :meth:`.PostgresNode.publish()` instead of direct + constructing publication objects. + + Args: + name: publication name. + node: publisher's node. + tables: tables list or None for all tables. + dbname: database name used to connect and perform subscription. + username: username used to connect to the database. + """ + self.name = name + self.node = node + self.dbname = dbname or default_dbname() + self.username = username or default_username() + + # create publication in database + t = "table " + ", ".join(tables) if tables else "all tables" + query = "create publication {} for {}" + node.execute(query.format(name, t), dbname=dbname, username=username) + + def drop(self, dbname=None, username=None): + """ + Drop publication + """ + self.node.execute("drop publication {}".format(self.name), + dbname=dbname, + username=username) + + def add_tables(self, tables, dbname=None, username=None): + """ + Add tables to the publication. Cannot be used if publication was + created with empty tables list. + + Args: + tables: a list of tables to be added to the publication. + """ + if not tables: + raise ValueError("Tables list is empty") + + query = "alter publication {} add table {}" + self.node.execute(query.format(self.name, ", ".join(tables)), + dbname=dbname or self.dbname, + username=username or self.username) + + +class Subscription(object): + def __init__(self, + node, + publication, + name=None, + dbname=None, + username=None, + **params): + """ + Constructor. Use :meth:`.PostgresNode.subscribe()` instead of direct + constructing subscription objects. + + Args: + name: subscription name. + node: subscriber's node. + publication: :class:`.Publication` object we are subscribing to + (see :meth:`.PostgresNode.publish()`). + dbname: database name used to connect and perform subscription. + username: username used to connect to the database. + params: subscription parameters (see documentation on `CREATE SUBSCRIPTION + `_ + for details). + """ + self.name = name + self.node = node + self.pub = publication + + # connection info + conninfo = { + "dbname": self.pub.dbname, + "user": self.pub.username, + "host": self.pub.node.host, + "port": self.pub.node.port + } + + query = ( + "create subscription {} connection '{}' publication {}").format( + name, options_string(**conninfo), self.pub.name) + + # additional parameters + if params: + query += " with ({})".format(options_string(**params)) + + # Note: cannot run 'create subscription' query in transaction mode + node.execute(query, dbname=dbname, username=username) + + def disable(self, dbname=None, username=None): + """ + Disables the running subscription. + """ + query = "alter subscription {} disable" + self.node.execute(query.format(self.name), dbname=None, username=None) + + def enable(self, dbname=None, username=None): + """ + Enables the previously disabled subscription. + """ + query = "alter subscription {} enable" + self.node.execute(query.format(self.name), dbname=None, username=None) + + def refresh(self, copy_data=True, dbname=None, username=None): + """ + Disables the running subscription. + """ + query = "alter subscription {} refresh publication with (copy_data={})" + self.node.execute(query.format(self.name, copy_data), + dbname=dbname, + username=username) + + def drop(self, dbname=None, username=None): + """ + Drops subscription + """ + self.node.execute("drop subscription {}".format(self.name), + dbname=dbname, + username=username) + + def catchup(self, username=None): + """ + Wait until subscription catches up with publication. + + Args: + username: remote node's user name. + """ + try: + pub_lsn = self.pub.node.execute(query="select pg_current_wal_lsn()", + dbname=None, + username=None)[0][0] # yapf: disable + # create dummy xact, as LR replicates only on commit. + self.pub.node.execute(query="select txid_current()", + dbname=None, + username=None) + query = """ + select '{}'::pg_lsn - replay_lsn <= 0 + from pg_catalog.pg_stat_replication where application_name = '{}' + """.format(pub_lsn, self.name) + + # wait until this LSN reaches subscriber + self.pub.node.poll_query_until( + query=query, + dbname=self.pub.dbname, + username=username or self.pub.username, + max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) + + # Now, wait until there are no tablesync workers: probably + # replay_lsn above was sent with changes of new tables just skipped; + # they will be eaten by tablesync workers. + query = """ + select count(*) = 0 from pg_subscription_rel where srsubstate != 'r' + """ + self.node.poll_query_until( + query=query, + dbname=self.pub.dbname, + username=username or self.pub.username, + max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) + except Exception as e: + raise_from(CatchUpException("Failed to catch up"), e) diff --git a/testgres/standby.py b/testgres/standby.py new file mode 100644 index 00000000..859f874e --- /dev/null +++ b/testgres/standby.py @@ -0,0 +1,49 @@ +# coding: utf-8 + +import six + + +@six.python_2_unicode_compatible +class First: + """ + Specifies a priority-based synchronous replication and makes transaction + commits wait until their WAL records are replicated to ``num_sync`` + synchronous standbys chosen based on their priorities. + + Args: + sync_num (int): the number of standbys that transaction need to wait + for replies from + standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby + nodes + """ + def __init__(self, sync_num, standbys): + self.sync_num = sync_num + self.standbys = standbys + + def __str__(self): + return u"{} ({})".format( + self.sync_num, + u", ".join(u"\"{}\"".format(r.name) for r in self.standbys)) + + +@six.python_2_unicode_compatible +class Any: + """ + Specifies a quorum-based synchronous replication and makes transaction + commits wait until their WAL records are replicated to at least ``num_sync`` + listed standbys. Only available for Postgres 10 and newer. + + Args: + sync_num (int): the number of standbys that transaction need to wait + for replies from + standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby + nodes + """ + def __init__(self, sync_num, standbys): + self.sync_num = sync_num + self.standbys = standbys + + def __str__(self): + return u"ANY {} ({})".format( + self.sync_num, + u", ".join(u"\"{}\"".format(r.name) for r in self.standbys)) diff --git a/testgres/utils.py b/testgres/utils.py index ada4c461..d231eec3 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -1,72 +1,70 @@ # coding: utf-8 from __future__ import division +from __future__ import print_function -import functools -import io import os -import port_for -import six -import subprocess -from distutils.version import LooseVersion +import sys + +from contextlib import contextmanager +from packaging.version import Version, InvalidVersion +import re + +from six import iteritems -from .config import TestgresConfig from .exceptions import ExecUtilException +from .config import testgres_config as tconf +from .operations.os_ops import OsOperations +from .operations.remote_ops import RemoteOperations +from .operations.local_ops import LocalOperations +from .operations.helpers import Helpers as OsHelpers + +from .impl.port_manager__generic import PortManager__Generic # rows returned by PG_CONFIG _pg_config_data = {} -# ports used by nodes -bound_ports = set() - +# +# The old, global "port manager" always worked with LOCAL system +# +_old_port_manager = PortManager__Generic(LocalOperations.get_single_instance()) -def reserve_port(): - """ - Generate a new port and add it to 'bound_ports'. - """ +# ports used by nodes +bound_ports = _old_port_manager._reserved_ports - port = port_for.select_random(exclude_ports=bound_ports) - bound_ports.add(port) - return port +# re-export version type +class PgVer(Version): + def __init__(self, version: str) -> None: + try: + super().__init__(version) + except InvalidVersion: + version = re.sub(r"[a-zA-Z].*", "", version) + super().__init__(version) -def release_port(port): +def internal__reserve_port(): """ - Free port provided by reserve_port(). + Generate a new port and add it to 'bound_ports'. """ - - bound_ports.remove(port) + return _old_port_manager.reserve_port() -def default_dbname(): +def internal__release_port(port): """ - Return default DB name. + Free port provided by reserve_port(). """ - return 'postgres' + assert type(port) == int # noqa: E721 + return _old_port_manager.release_port(port) -def default_username(): - """ - Return default username (current user). - """ - - import getpass - return getpass.getuser() +reserve_port = internal__reserve_port +release_port = internal__release_port -def generate_app_name(): - """ - Generate a new application name for node. - """ - - import uuid - return 'testgres-{}'.format(str(uuid.uuid4())) - - -def execute_utility(args, logfile): +def execute_utility(args, logfile=None, verbose=False): """ Execute utility (pg_ctl, pg_dump etc). @@ -77,42 +75,47 @@ def execute_utility(args, logfile): Returns: stdout of executed utility. """ - - # run utility - process = subprocess.Popen( - args, # util + params - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - # get result and decode it - out, _ = process.communicate() - out = '' if not out else out.decode('utf-8') + return execute_utility2(tconf.os_ops, args, logfile, verbose) + + +def execute_utility2( + os_ops: OsOperations, + args, + logfile=None, + verbose=False, + ignore_errors=False, + exec_env=None, +): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + assert type(verbose) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + exit_status, out, error = os_ops.exec_command( + args, + verbose=True, + ignore_errors=ignore_errors, + encoding=OsHelpers.GetDefaultEncoding(), + exec_env=exec_env) + + out = '' if not out else out # write new log entry if possible - try: - with io.open(logfile, 'a') as file_out: - # write util's name and args - file_out.write(u' '.join(args)) - - # write output + if logfile: + try: + os_ops.write(filename=logfile, data=args, truncate=True) if out: - file_out.write(u'\n') - file_out.write(out) - - # finally, a separator - file_out.write(u'\n') - except IOError: - pass - - # format exception, if needed - error_code = process.returncode - if error_code: - error_text = (u"{} failed with exit code {}\n" - u"log:\n----\n{}\n").format(args[0], error_code, out) - - raise ExecUtilException(error_text, error_code) - - return out + # comment-out lines + lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] + os_ops.write(filename=logfile, data=lines) + except IOError: + raise ExecUtilException( + "Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) + if verbose: + return exit_status, out, error + else: + return out def get_bin_path(filename): @@ -120,34 +123,58 @@ def get_bin_path(filename): Return absolute path to an executable using PG_BIN or PG_CONFIG. This function does nothing if 'filename' is already absolute. """ + return get_bin_path2(tconf.os_ops, filename) + + +def get_bin_path2(os_ops: OsOperations, filename): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) # check if it's already absolute if os.path.isabs(filename): return filename + if isinstance(os_ops, RemoteOperations): + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + else: + # try PG_CONFIG - get from local machine + pg_config = os.environ.get("PG_CONFIG") - # try PG_CONFIG - pg_config = os.environ.get("PG_CONFIG") if pg_config: - bindir = get_pg_config()["BINDIR"] + bindir = get_pg_config(pg_config, os_ops)["BINDIR"] return os.path.join(bindir, filename) # try PG_BIN - pg_bin = os.environ.get("PG_BIN") + pg_bin = os_ops.environ("PG_BIN") if pg_bin: return os.path.join(pg_bin, filename) + pg_config_path = os_ops.find_executable('pg_config') + if pg_config_path: + bindir = get_pg_config(pg_config_path)["BINDIR"] + return os.path.join(bindir, filename) + return filename -def get_pg_config(): +def get_pg_config(pg_config_path=None, os_ops=None): """ Return output of pg_config (provided that it is installed). - NOTE: this fuction caches the result by default (see TestgresConfig). + NOTE: this function caches the result by default (see GlobalConfig). """ + if os_ops is None: + os_ops = tconf.os_ops + + return get_pg_config2(os_ops, pg_config_path) + + +def get_pg_config2(os_ops: OsOperations, pg_config_path): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + def cache_pg_config_data(cmd): # execute pg_config and get the output - out = subprocess.check_output([cmd]).decode('utf-8') + out = os_ops.exec_command(cmd, encoding='utf-8') data = {} for line in out.splitlines(): @@ -162,16 +189,24 @@ def cache_pg_config_data(cmd): return data # drop cache if asked to - if not TestgresConfig.cache_pg_config: + if not tconf.cache_pg_config: global _pg_config_data _pg_config_data = {} # return cached data - if _pg_config_data: + if not pg_config_path and _pg_config_data: return _pg_config_data - # try PG_CONFIG - pg_config = os.environ.get("PG_CONFIG") + # try specified pg_config path or PG_CONFIG + if pg_config_path: + return cache_pg_config_data(pg_config_path) + + if isinstance(os_ops, RemoteOperations): + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + else: + # try PG_CONFIG - get from local machine + pg_config = os.environ.get("PG_CONFIG") + if pg_config: return cache_pg_config_data(pg_config) @@ -185,33 +220,39 @@ def cache_pg_config_data(cmd): return cache_pg_config_data("pg_config") -def get_pg_version(): +def get_pg_version2(os_ops: OsOperations, bin_dir=None): """ Return PostgreSQL version provided by postmaster. """ + assert os_ops is not None + assert isinstance(os_ops, OsOperations) - # get raw version (e.g. postgres (PostgreSQL) 9.5.7) - _params = [get_bin_path('postgres'), '--version'] - raw_ver = subprocess.check_output(_params).decode('utf-8') + # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) + postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path2(os_ops, 'postgres') + cmd = [postgres_path, '--version'] + raw_ver = os_ops.exec_command(cmd, encoding='utf-8') - # cook version of PostgreSQL - version = raw_ver.strip().split(' ')[-1] \ - .partition('devel')[0] \ - .partition('beta')[0] \ - .partition('rc')[0] + return parse_pg_version(raw_ver) - return version - -def pg_version_ge(version): +def get_pg_version(bin_dir=None): """ - Check if PostgreSQL is 'version' or newer. + Return PostgreSQL version provided by postmaster. """ - cur_ver = LooseVersion(get_pg_version()) - min_ver = LooseVersion(version) + return get_pg_version2(tconf.os_ops, bin_dir) + + +def parse_pg_version(version_out): + # Generalize removal of system-specific suffixes (anything in parentheses) + raw_ver = re.sub(r'\([^)]*\)', '', version_out).strip() - return cur_ver >= min_ver + # Cook version of PostgreSQL + version = raw_ver.split(' ')[-1] \ + .partition('devel')[0] \ + .partition('beta')[0] \ + .partition('rc')[0] + return version def file_tail(f, num_lines): @@ -241,73 +282,27 @@ def file_tail(f, num_lines): buffers = int(buffers * max(2, num_lines / max(cur_lines, 1))) -def positional_args_hack(*special_cases): +def eprint(*args, **kwargs): """ - Convert positional args described by - 'special_cases' into named args. - - Example: - @positional_args_hack(['abc'], ['def', 'abc']) - def some_api_func(...) - - This is useful for compatibility. + Print stuff to stderr. """ + print(*args, file=sys.stderr, **kwargs) - cases = dict() - - for case in special_cases: - k = len(case) - assert k not in six.iterkeys(cases), 'len must be unique' - cases[k] = case - - def decorator(function): - @functools.wraps(function) - def wrapper(*args, **kwargs): - k = len(args) - - if k in six.iterkeys(cases): - case = cases[k] - - for i in range(0, k): - arg_name = case[i] - arg_val = args[i] - - # transform into named - kwargs[arg_name] = arg_val - - # get rid of them - args = [] - - return function(*args, **kwargs) - return wrapper +def options_string(separator=u" ", **kwargs): + return separator.join(u"{}={}".format(k, v) for k, v in iteritems(kwargs)) - return decorator - -def method_decorator(decorator): +@contextmanager +def clean_on_error(node): """ - Convert a function decorator into a method decorator. + Context manager to wrap PostgresNode and such. + Calls cleanup() method when underlying code raises an exception. """ - def _dec(func): - def _wrapper(self, *args, **kwargs): - @decorator - def bound_func(*args2, **kwargs2): - return func.__get__(self, type(self))(*args2, **kwargs2) - - # 'bound_func' is a closure and can see 'self' - return bound_func(*args, **kwargs) - - # preserve docs - functools.update_wrapper(_wrapper, func) - - return _wrapper - - # preserve docs - functools.update_wrapper(_dec, decorator) - - # change name for easier debugging - _dec.__name__ = 'method_decorator({})'.format(decorator.__name__) - - return _dec + try: + yield node + except Exception: + # TODO: should we wrap this in try-block? + node.cleanup() + raise diff --git a/tests/README.md b/tests/README.md index 5a5ecd07..d89efc7e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -21,9 +21,38 @@ export PG_BIN=/path/to/pg/bin ```bash # Set path to PostgreSQL and python version -export PATH=$PATH:/path/to/pg/bin +export PATH=/path/to/pg/bin:$PATH export PYTHON_VERSION=3 # or 2 # Run tests ./run_tests.sh ``` + + +#### Remote host tests + +1. Start remote host or docker container +2. Make sure that you run ssh +```commandline +sudo apt-get install openssh-server +sudo systemctl start sshd +``` +3. You need to connect to the remote host at least once to add it to the known hosts file +4. Generate ssh keys +5. Set up params for tests + + +```commandline +conn_params = ConnectionParams( + host='remote_host', + username='username', + ssh_key=/path/to/your/ssh/key' +) +os_ops = RemoteOperations(conn_params) +``` +If you have different path to `PG_CONFIG` on your local and remote host you can set up `PG_CONFIG_REMOTE`, this value will be +using during work with remote host. + +`test_remote` - Tests for RemoteOperations class. + +`test_simple_remote` - Tests that create node and check it. The same as `test_simple`, but for remote node. \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..111edc87 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,989 @@ +# ///////////////////////////////////////////////////////////////////////////// +# PyTest Configuration + +import pluggy +import pytest +import os +import logging +import pathlib +import math +import datetime +import typing + +import _pytest.outcomes +import _pytest.unittest +import _pytest.logging + +# ///////////////////////////////////////////////////////////////////////////// + +C_ROOT_DIR__RELATIVE = ".." + +# ///////////////////////////////////////////////////////////////////////////// +# TestConfigPropNames + + +class TestConfigPropNames: + TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" + + +# ///////////////////////////////////////////////////////////////////////////// + +T_TUPLE__str_int = typing.Tuple[str, int] + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData__Helper + + +class TestStartupData__Helper: + sm_StartTS = datetime.datetime.now() + + # -------------------------------------------------------------------- + def GetStartTS() -> datetime.datetime: + assert type(__class__.sm_StartTS) == datetime.datetime # noqa: E721 + return __class__.sm_StartTS + + # -------------------------------------------------------------------- + def CalcRootDir() -> str: + r = os.path.abspath(__file__) + r = os.path.dirname(r) + r = os.path.join(r, C_ROOT_DIR__RELATIVE) + r = os.path.abspath(r) + return r + + # -------------------------------------------------------------------- + def CalcRootLogDir() -> str: + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + else: + rootDir = __class__.CalcRootDir() + resultPath = os.path.join(rootDir, "logs") + + assert type(resultPath) == str # noqa: E721 + return resultPath + + # -------------------------------------------------------------------- + def CalcCurrentTestWorkerSignature() -> str: + currentPID = os.getpid() + assert type(currentPID) + + startTS = __class__.sm_StartTS + assert type(startTS) + + result = "pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format( + startTS.year, + startTS.month, + startTS.day, + startTS.hour, + startTS.minute, + startTS.second, + ) + + gwid = os.environ.get("PYTEST_XDIST_WORKER") + + if gwid is not None: + result += "--xdist_" + str(gwid) + + result += "--" + "pid" + str(currentPID) + return result + + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData + + +class TestStartupData: + sm_RootDir: str = TestStartupData__Helper.CalcRootDir() + sm_CurrentTestWorkerSignature: str = ( + TestStartupData__Helper.CalcCurrentTestWorkerSignature() + ) + + sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir() + + # -------------------------------------------------------------------- + def GetRootDir() -> str: + assert type(__class__.sm_RootDir) == str # noqa: E721 + return __class__.sm_RootDir + + # -------------------------------------------------------------------- + def GetRootLogDir() -> str: + assert type(__class__.sm_RootLogDir) == str # noqa: E721 + return __class__.sm_RootLogDir + + # -------------------------------------------------------------------- + def GetCurrentTestWorkerSignature() -> str: + assert type(__class__.sm_CurrentTestWorkerSignature) == str # noqa: E721 + return __class__.sm_CurrentTestWorkerSignature + + +# ///////////////////////////////////////////////////////////////////////////// +# TEST_PROCESS_STATS + + +class TEST_PROCESS_STATS: + cTotalTests: int = 0 + cNotExecutedTests: int = 0 + cExecutedTests: int = 0 + cPassedTests: int = 0 + cFailedTests: int = 0 + cXFailedTests: int = 0 + cSkippedTests: int = 0 + cNotXFailedTests: int = 0 + cWarningTests: int = 0 + cUnexpectedTests: int = 0 + cAchtungTests: int = 0 + + FailedTests: typing.List[T_TUPLE__str_int] = list() + XFailedTests: typing.List[T_TUPLE__str_int] = list() + NotXFailedTests: typing.List[str] = list() + WarningTests: typing.List[T_TUPLE__str_int] = list() + AchtungTests: typing.List[str] = list() + + cTotalDuration: datetime.timedelta = datetime.timedelta() + + cTotalErrors: int = 0 + cTotalWarnings: int = 0 + + # -------------------------------------------------------------------- + def incrementTotalTestCount() -> None: + assert type(__class__.cTotalTests) == int # noqa: E721 + assert __class__.cTotalTests >= 0 + + __class__.cTotalTests += 1 + + assert __class__.cTotalTests > 0 + + # -------------------------------------------------------------------- + def incrementNotExecutedTestCount() -> None: + assert type(__class__.cNotExecutedTests) == int # noqa: E721 + assert __class__.cNotExecutedTests >= 0 + + __class__.cNotExecutedTests += 1 + + assert __class__.cNotExecutedTests > 0 + + # -------------------------------------------------------------------- + def incrementExecutedTestCount() -> int: + assert type(__class__.cExecutedTests) == int # noqa: E721 + assert __class__.cExecutedTests >= 0 + + __class__.cExecutedTests += 1 + + assert __class__.cExecutedTests > 0 + return __class__.cExecutedTests + + # -------------------------------------------------------------------- + def incrementPassedTestCount() -> None: + assert type(__class__.cPassedTests) == int # noqa: E721 + assert __class__.cPassedTests >= 0 + + __class__.cPassedTests += 1 + + assert __class__.cPassedTests > 0 + + # -------------------------------------------------------------------- + def incrementFailedTestCount(testID: str, errCount: int) -> None: + assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount > 0 + assert type(__class__.FailedTests) == list # noqa: E721 + assert type(__class__.cFailedTests) == int # noqa: E721 + assert __class__.cFailedTests >= 0 + + __class__.FailedTests.append((testID, errCount)) # raise? + __class__.cFailedTests += 1 + + assert len(__class__.FailedTests) > 0 + assert __class__.cFailedTests > 0 + assert len(__class__.FailedTests) == __class__.cFailedTests + + # -------- + assert type(__class__.cTotalErrors) == int # noqa: E721 + assert __class__.cTotalErrors >= 0 + + __class__.cTotalErrors += errCount + + assert __class__.cTotalErrors > 0 + + # -------------------------------------------------------------------- + def incrementXFailedTestCount(testID: str, errCount: int) -> None: + assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount >= 0 + assert type(__class__.XFailedTests) == list # noqa: E721 + assert type(__class__.cXFailedTests) == int # noqa: E721 + assert __class__.cXFailedTests >= 0 + + __class__.XFailedTests.append((testID, errCount)) # raise? + __class__.cXFailedTests += 1 + + assert len(__class__.XFailedTests) > 0 + assert __class__.cXFailedTests > 0 + assert len(__class__.XFailedTests) == __class__.cXFailedTests + + # -------------------------------------------------------------------- + def incrementSkippedTestCount() -> None: + assert type(__class__.cSkippedTests) == int # noqa: E721 + assert __class__.cSkippedTests >= 0 + + __class__.cSkippedTests += 1 + + assert __class__.cSkippedTests > 0 + + # -------------------------------------------------------------------- + def incrementNotXFailedTests(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.NotXFailedTests) == list # noqa: E721 + assert type(__class__.cNotXFailedTests) == int # noqa: E721 + assert __class__.cNotXFailedTests >= 0 + + __class__.NotXFailedTests.append(testID) # raise? + __class__.cNotXFailedTests += 1 + + assert len(__class__.NotXFailedTests) > 0 + assert __class__.cNotXFailedTests > 0 + assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + + # -------------------------------------------------------------------- + def incrementWarningTestCount(testID: str, warningCount: int) -> None: + assert type(testID) == str # noqa: E721 + assert type(warningCount) == int # noqa: E721 + assert testID != "" + assert warningCount > 0 + assert type(__class__.WarningTests) == list # noqa: E721 + assert type(__class__.cWarningTests) == int # noqa: E721 + assert __class__.cWarningTests >= 0 + + __class__.WarningTests.append((testID, warningCount)) # raise? + __class__.cWarningTests += 1 + + assert len(__class__.WarningTests) > 0 + assert __class__.cWarningTests > 0 + assert len(__class__.WarningTests) == __class__.cWarningTests + + # -------- + assert type(__class__.cTotalWarnings) == int # noqa: E721 + assert __class__.cTotalWarnings >= 0 + + __class__.cTotalWarnings += warningCount + + assert __class__.cTotalWarnings > 0 + + # -------------------------------------------------------------------- + def incrementUnexpectedTests() -> None: + assert type(__class__.cUnexpectedTests) == int # noqa: E721 + assert __class__.cUnexpectedTests >= 0 + + __class__.cUnexpectedTests += 1 + + assert __class__.cUnexpectedTests > 0 + + # -------------------------------------------------------------------- + def incrementAchtungTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.AchtungTests) == list # noqa: E721 + assert type(__class__.cAchtungTests) == int # noqa: E721 + assert __class__.cAchtungTests >= 0 + + __class__.AchtungTests.append(testID) # raise? + __class__.cAchtungTests += 1 + + assert len(__class__.AchtungTests) > 0 + assert __class__.cAchtungTests > 0 + assert len(__class__.AchtungTests) == __class__.cAchtungTests + + +# ///////////////////////////////////////////////////////////////////////////// + + +def timedelta_to_human_text(delta: datetime.timedelta) -> str: + assert isinstance(delta, datetime.timedelta) + + C_SECONDS_IN_MINUTE = 60 + C_SECONDS_IN_HOUR = 60 * C_SECONDS_IN_MINUTE + + v = delta.seconds + + cHours = int(v / C_SECONDS_IN_HOUR) + v = v - cHours * C_SECONDS_IN_HOUR + cMinutes = int(v / C_SECONDS_IN_MINUTE) + cSeconds = v - cMinutes * C_SECONDS_IN_MINUTE + + result = "" if delta.days == 0 else "{0} day(s) ".format(delta.days) + + result = result + "{:02d}:{:02d}:{:02d}.{:06d}".format( + cHours, cMinutes, cSeconds, delta.microseconds + ) + + return result + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__build_test_id(item: pytest.Function) -> str: + assert item is not None + assert isinstance(item, pytest.Function) + + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + return testID + + +# ///////////////////////////////////////////////////////////////////////////// + +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() +g_critical_msg_count_key = pytest.StashKey[int]() + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__makereport__setup( + item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result +): + assert item is not None + assert call is not None + assert outcome is not None + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) + assert type(call) == pytest.CallInfo # noqa: E721 + assert type(outcome) == pluggy.Result # noqa: E721 + + C_LINE1 = "******************************************************" + + # logging.info("pytest_runtest_makereport - setup") + + TEST_PROCESS_STATS.incrementTotalTestCount() + + rep: pytest.TestReport = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + if rep.outcome == "skipped": + TEST_PROCESS_STATS.incrementNotExecutedTestCount() + return + + testID = helper__build_test_id(item) + + if rep.outcome == "passed": + testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() + + logging.info(C_LINE1) + logging.info("* START TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Number: {0}".format(testNumber)) + logging.info("*") + return + + assert rep.outcome != "passed" + + TEST_PROCESS_STATS.incrementAchtungTestCount(testID) + + logging.info(C_LINE1) + logging.info("* ACHTUNG TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Outcome is [{0}]".format(rep.outcome)) + + if rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + logging.info("*") + logging.error(call.excinfo.value) + + logging.info("*") + return + + +# ------------------------------------------------------------------------ +def helper__makereport__call( + item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result +): + assert item is not None + assert call is not None + assert outcome is not None + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) + assert type(call) == pytest.CallInfo # noqa: E721 + assert type(outcome) == pluggy.Result # noqa: E721 + + # -------- + item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count1) == int # noqa: E721 + assert item_error_msg_count1 >= 0 + + item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0) + assert type(item_error_msg_count2) == int # noqa: E721 + assert item_error_msg_count2 >= 0 + + item_error_msg_count = item_error_msg_count1 + item_error_msg_count2 + + # -------- + item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) + assert type(item_warning_msg_count) == int # noqa: E721 + assert item_warning_msg_count >= 0 + + # -------- + rep = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + # -------- + testID = helper__build_test_id(item) + + # -------- + assert call.start <= call.stop + + startDT = datetime.datetime.fromtimestamp(call.start) + assert type(startDT) == datetime.datetime # noqa: E721 + stopDT = datetime.datetime.fromtimestamp(call.stop) + assert type(stopDT) == datetime.datetime # noqa: E721 + + testDurration = stopDT - startDT + assert type(testDurration) == datetime.timedelta # noqa: E721 + + # -------- + exitStatus = None + if rep.outcome == "skipped": + assert call.excinfo is not None # research + assert call.excinfo.value is not None # research + + if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 + assert not hasattr(rep, "wasxfail") + + exitStatus = "SKIPPED" + reasonText = str(call.excinfo.value) + reasonMsgTempl = "SKIP REASON: {0}" + + TEST_PROCESS_STATS.incrementSkippedTestCount() + + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 + exitStatus = "XFAILED" + reasonText = str(call.excinfo.value) + reasonMsgTempl = "XFAIL REASON: {0}" + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + + else: + exitStatus = "XFAILED" + assert hasattr(rep, "wasxfail") + assert rep.wasxfail is not None + assert type(rep.wasxfail) == str # noqa: E721 + + reasonText = rep.wasxfail + reasonMsgTempl = "XFAIL REASON: {0}" + + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + + assert type(reasonText) == str # noqa: E721 + + if reasonText != "": + assert type(reasonMsgTempl) == str # noqa: E721 + logging.info("*") + logging.info("* " + reasonMsgTempl.format(reasonText)) + + elif rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + assert item_error_msg_count > 0 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + assert item_error_msg_count > 0 + TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) + + exitStatus = "FAILED" + elif rep.outcome == "passed": + assert call.excinfo is None + + if hasattr(rep, "wasxfail"): + assert type(rep.wasxfail) == str # noqa: E721 + + TEST_PROCESS_STATS.incrementNotXFailedTests(testID) + + warnMsg = "NOTE: Test is marked as xfail" + + if rep.wasxfail != "": + warnMsg += " [" + rep.wasxfail + "]" + + logging.info(warnMsg) + exitStatus = "NOT XFAILED" + else: + assert not hasattr(rep, "wasxfail") + + TEST_PROCESS_STATS.incrementPassedTestCount() + exitStatus = "PASSED" + else: + TEST_PROCESS_STATS.incrementUnexpectedTests() + exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) + # [2025-03-28] It may create a useless problem in new environment. + # assert False + + # -------- + if item_warning_msg_count > 0: + TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + + # -------- + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + assert type(testDurration) == datetime.timedelta # noqa: E721 + + TEST_PROCESS_STATS.cTotalDuration += testDurration + + assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + + # -------- + logging.info("*") + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("*") + logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) + logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) + logging.info("*") + logging.info("* STOP TEST {0}".format(testID)) + logging.info("*") + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): + # + # https://docs.pytest.org/en/7.1.x/how-to/writing_hook_functions.html#hookwrapper-executing-around-other-hooks + # + # Note that hook wrappers donโ€™t return results themselves, + # they merely perform tracing or other side effects around the actual hook implementations. + # + # https://docs.pytest.org/en/7.1.x/reference/reference.html#test-running-runtest-hooks + # + assert item is not None + assert call is not None + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) + assert type(call) == pytest.CallInfo # noqa: E721 + + outcome: pluggy.Result = yield + assert outcome is not None + assert type(outcome) == pluggy.Result # noqa: E721 + + assert type(call.when) == str # noqa: E721 + + if call.when == "collect": + return + + if call.when == "setup": + helper__makereport__setup(item, call, outcome) + return + + if call.when == "call": + helper__makereport__call(item, call, outcome) + return + + if call.when == "teardown": + return + + errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format( + call.when + ) + + raise RuntimeError(errMsg) + + +# ///////////////////////////////////////////////////////////////////////////// + + +class LogWrapper2: + _old_method: any + _err_counter: typing.Optional[int] + _warn_counter: typing.Optional[int] + + _critical_counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._err_counter = None + self._warn_counter = None + + self._critical_counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._err_counter is None + assert self._warn_counter is None + + assert self._critical_counter is None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + + self._old_method = logging.root.handle + self._err_counter = 0 + self._warn_counter = 0 + + self._critical_counter = 0 + + logging.root.handle = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._err_counter is not None + assert self._warn_counter is not None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + + assert logging.root.handle is self + + logging.root.handle = self._old_method + + self._old_method = None + self._err_counter = None + self._warn_counter = None + self._critical_counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, record: logging.LogRecord): + assert record is not None + assert isinstance(record, logging.LogRecord) + assert self._old_method is not None + assert self._err_counter is not None + assert self._warn_counter is not None + assert self._critical_counter is not None + + assert type(self._err_counter) == int # noqa: E721 + assert self._err_counter >= 0 + assert type(self._warn_counter) == int # noqa: E721 + assert self._warn_counter >= 0 + assert type(self._critical_counter) == int # noqa: E721 + assert self._critical_counter >= 0 + + r = self._old_method(record) + + if record.levelno == logging.ERROR: + self._err_counter += 1 + assert self._err_counter > 0 + elif record.levelno == logging.WARNING: + self._warn_counter += 1 + assert self._warn_counter > 0 + elif record.levelno == logging.CRITICAL: + self._critical_counter += 1 + assert self._critical_counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class SIGNAL_EXCEPTION(Exception): + def __init__(self): + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + assert pyfuncitem is not None + assert isinstance(pyfuncitem, pytest.Function) + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + assert logging.root.handle is not None + + debug__log_handle_method = logging.root.handle + assert debug__log_handle_method is not None + + debug__log_error_method = logging.error + assert debug__log_error_method is not None + + debug__log_warning_method = logging.warning + assert debug__log_warning_method is not None + + pyfuncitem.stash[g_error_msg_count_key] = 0 + pyfuncitem.stash[g_warning_msg_count_key] = 0 + pyfuncitem.stash[g_critical_msg_count_key] = 0 + + try: + with LogWrapper2() as logWrapper: + assert type(logWrapper) == LogWrapper2 # noqa: E721 + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter == 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter == 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter == 0 + assert logging.root.handle is logWrapper + + r: pluggy.Result = yield + + assert r is not None + assert type(r) == pluggy.Result # noqa: E721 + + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter >= 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter >= 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter >= 0 + assert logging.root.handle is logWrapper + + assert g_error_msg_count_key in pyfuncitem.stash + assert g_warning_msg_count_key in pyfuncitem.stash + assert g_critical_msg_count_key in pyfuncitem.stash + + assert pyfuncitem.stash[g_error_msg_count_key] == 0 + assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + assert pyfuncitem.stash[g_critical_msg_count_key] == 0 + + pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter + pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter + pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter + + if r.exception is not None: + pass + elif logWrapper._err_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + elif logWrapper._critical_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + finally: + assert logging.error is debug__log_error_method + assert logging.warning is debug__log_warning_method + assert logging.root.handle == debug__log_handle_method + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__calc_W(n: int) -> int: + assert n > 0 + + x = int(math.log10(n)) + assert type(x) == int # noqa: E721 + assert x >= 0 + x += 1 + return x + + +# ------------------------------------------------------------------------ +def helper__print_test_list(tests: typing.List[str]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1}" + + nTest = 0 + + for t in tests: + assert type(t) == str # noqa: E721 + assert t != "" + nTest += 1 + logging.info(templateLine.format(nTest, t)) + + +# ------------------------------------------------------------------------ +def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1} ({2})" + + nTest = 0 + + for t in tests: + assert type(t) == tuple # noqa: E721 + assert len(t) == 2 + assert type(t[0]) == str # noqa: E721 + assert type(t[1]) == int # noqa: E721 + assert t[0] != "" + assert t[1] >= 0 + nTest += 1 + logging.info(templateLine.format(nTest, t[0], t[1])) + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.fixture(autouse=True, scope="session") +def run_after_tests(request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + yield + + C_LINE1 = "---------------------------" + + def LOCAL__print_line1_with_header(header: str): + assert type(C_LINE1) == str # noqa: E721 + assert type(header) == str # noqa: E721 + assert header != "" + logging.info(C_LINE1 + " [" + header + "]") + + def LOCAL__print_test_list(header: str, test_count: int, test_list: typing.List[str]): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list(test_list) + logging.info("") + + def LOCAL__print_test_list2( + header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int] + ): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list2(test_list) + logging.info("") + + # fmt: off + LOCAL__print_test_list( + "ACHTUNG TESTS", + TEST_PROCESS_STATS.cAchtungTests, + TEST_PROCESS_STATS.AchtungTests, + ) + + LOCAL__print_test_list2( + "FAILED TESTS", + TEST_PROCESS_STATS.cFailedTests, + TEST_PROCESS_STATS.FailedTests + ) + + LOCAL__print_test_list2( + "XFAILED TESTS", + TEST_PROCESS_STATS.cXFailedTests, + TEST_PROCESS_STATS.XFailedTests, + ) + + LOCAL__print_test_list( + "NOT XFAILED TESTS", + TEST_PROCESS_STATS.cNotXFailedTests, + TEST_PROCESS_STATS.NotXFailedTests, + ) + + LOCAL__print_test_list2( + "WARNING TESTS", + TEST_PROCESS_STATS.cWarningTests, + TEST_PROCESS_STATS.WarningTests, + ) + # fmt: on + + LOCAL__print_line1_with_header("SUMMARY STATISTICS") + logging.info("") + logging.info("[TESTS]") + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED : {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info("") + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" WITH WARNINGS: {0}".format(TEST_PROCESS_STATS.cWarningTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info("") + + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + + LOCAL__print_line1_with_header("TIME") + logging.info("") + logging.info( + " TOTAL DURATION: {0}".format( + timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration) + ) + ) + logging.info("") + + LOCAL__print_line1_with_header("TOTAL INFORMATION") + logging.info("") + logging.info(" TOTAL ERROR COUNT : {0}".format(TEST_PROCESS_STATS.cTotalErrors)) + logging.info(" TOTAL WARNING COUNT: {0}".format(TEST_PROCESS_STATS.cTotalWarnings)) + logging.info("") + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + assert isinstance(config, pytest.Config) + + log_name = TestStartupData.GetCurrentTestWorkerSignature() + log_name += ".log" + + log_dir = TestStartupData.GetRootLogDir() + + pathlib.Path(log_dir).mkdir(exist_ok=True) + + logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( + "logging-plugin" + ) + + assert logging_plugin is not None + assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) + + logging_plugin.set_log_path(os.path.join(log_dir, log_name)) + + +# ///////////////////////////////////////////////////////////////////////////// diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py new file mode 100644 index 00000000..f3df41a3 --- /dev/null +++ b/tests/helpers/global_data.py @@ -0,0 +1,78 @@ +from testgres.operations.os_ops import OsOperations +from testgres.operations.os_ops import ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations + +from testgres.node import PortManager +from testgres.node import PortManager__ThisHost +from testgres.node import PortManager__Generic + +import os + + +class OsOpsDescr: + sign: str + os_ops: OsOperations + + def __init__(self, sign: str, os_ops: OsOperations): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + self.sign = sign + self.os_ops = os_ops + + +class OsOpsDescrs: + sm_remote_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) + + sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) + + sm_local_os_ops = LocalOperations.get_single_instance() + + sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) + + +class PortManagers: + sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) + + sm_local_port_manager = PortManager__ThisHost.get_single_instance() + + sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) + + +class PostgresNodeService: + sign: str + os_ops: OsOperations + port_manager: PortManager + + def __init__(self, sign: str, os_ops: OsOperations, port_manager: PortManager): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + assert isinstance(port_manager, PortManager) + self.sign = sign + self.os_ops = os_ops + self.port_manager = port_manager + + +class PostgresNodeServices: + sm_remote = PostgresNodeService( + "remote", + OsOpsDescrs.sm_remote_os_ops, + PortManagers.sm_remote_port_manager + ) + + sm_local = PostgresNodeService( + "local", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local_port_manager + ) + + sm_local2 = PostgresNodeService( + "local2", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local2_port_manager + ) diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py new file mode 100644 index 00000000..11357c30 --- /dev/null +++ b/tests/helpers/run_conditions.py @@ -0,0 +1,12 @@ +# coding: utf-8 +import pytest +import platform + + +class RunConditions: + # It is not a test kit! + __test__ = False + + def skip_if_windows(): + if platform.system().lower() == "windows": + pytest.skip("This test does not support Windows.") diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..a80a11f1 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,41 @@ +from testgres import TestgresConfig +from testgres import configure_testgres +from testgres import scoped_config +from testgres import pop_config + +import testgres + +import pytest + + +class TestConfig: + def test_config_stack(self): + # no such option + with pytest.raises(expected_exception=TypeError): + configure_testgres(dummy=True) + + # we have only 1 config in stack + with pytest.raises(expected_exception=IndexError): + pop_config() + + d0 = TestgresConfig.cached_initdb_dir + d1 = 'dummy_abc' + d2 = 'dummy_def' + + with scoped_config(cached_initdb_dir=d1) as c1: + assert (c1.cached_initdb_dir == d1) + + with scoped_config(cached_initdb_dir=d2) as c2: + stack_size = len(testgres.config.config_stack) + + # try to break a stack + with pytest.raises(expected_exception=TypeError): + with scoped_config(dummy=True): + pass + + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) + + assert (c1.cached_initdb_dir == d1) + + assert (TestgresConfig.cached_initdb_dir == d0) diff --git a/tests/test_conftest.py--devel b/tests/test_conftest.py--devel new file mode 100644 index 00000000..67c1dafe --- /dev/null +++ b/tests/test_conftest.py--devel @@ -0,0 +1,80 @@ +import pytest +import logging + + +class TestConfest: + def test_failed(self): + raise Exception("TEST EXCEPTION!") + + def test_ok(self): + pass + + @pytest.mark.skip() + def test_mark_skip__no_reason(self): + pass + + @pytest.mark.xfail() + def test_mark_xfail__no_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail() + def test_mark_xfail__no_reason___no_error(self): + pass + + @pytest.mark.skip(reason="reason") + def test_mark_skip__with_reason(self): + pass + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason___no_error(self): + pass + + def test_exc_skip__no_reason(self): + pytest.skip() + + def test_exc_xfail__no_reason(self): + pytest.xfail() + + def test_exc_skip__with_reason(self): + pytest.skip(reason="SKIP REASON") + + def test_exc_xfail__with_reason(self): + pytest.xfail(reason="XFAIL EXCEPTION") + + def test_log_error(self): + logging.error("IT IS A LOG ERROR!") + + def test_log_error_and_exc(self): + logging.error("IT IS A LOG ERROR!") + + raise Exception("TEST EXCEPTION!") + + def test_log_error_and_warning(self): + logging.error("IT IS A LOG ERROR!") + logging.warning("IT IS A LOG WARNING!") + logging.error("IT IS THE SECOND LOG ERROR!") + logging.warning("IT IS THE SECOND LOG WARNING!") + + @pytest.mark.xfail() + def test_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail() + def test_two_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_two_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py new file mode 100644 index 00000000..5ae3a61f --- /dev/null +++ b/tests/test_os_ops_common.py @@ -0,0 +1,1115 @@ +# coding: utf-8 +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations +from .helpers.run_conditions import RunConditions + +import os + +import pytest +import re +import tempfile +import logging +import socket +import threading +import typing +import uuid + +from testgres import InvalidOperationException +from testgres import ExecUtilException + +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future as ThreadFuture + + +class TestOsOpsCommon: + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_exec_command_success(self, os_ops: OsOperations): + """ + Test exec_command for successful command execution. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "python3 --version"] + + response = os_ops.exec_command(cmd) + + assert b'Python 3.' in response + + def test_exec_command_failure(self, os_ops: OsOperations): + """ + Test exec_command for command execution failure. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "nonexistent_command"] + + while True: + try: + os_ops.exec_command(cmd) + except ExecUtilException as e: + assert type(e.exit_code) == int # noqa: E721 + assert e.exit_code == 127 + + assert type(e.message) == str # noqa: E721 + assert type(e.error) == bytes # noqa: E721 + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error + break + raise Exception("We wait an exception!") + + def test_exec_command_failure__expect_error(self, os_ops: OsOperations): + """ + Test exec_command for command execution failure. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "nonexistent_command"] + + exit_status, result, error = os_ops.exec_command(cmd, verbose=True, expect_error=True) + + assert exit_status == 127 + assert result == b'' + assert type(error) == bytes # noqa: E721 + assert b"nonexistent_command" in error + assert b"not found" in error + + def test_exec_command_with_exec_env(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "TESTGRES_TEST__EXEC_ENV_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: "Hello!"} + + response = os_ops.exec_command(cmd, exec_env=exec_env) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'Hello!\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'\n' + + def test_exec_command__test_unset(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "LANG" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + response1 = os_ops.exec_command(cmd) + assert response1 is not None + assert type(response1) == bytes # noqa: E721 + + if response1 == b'\n': + logging.warning("Environment variable {} is not defined.".format(C_ENV_NAME)) + return + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) == bytes # noqa: E721 + assert response2 == b'\n' + + response3 = os_ops.exec_command(cmd) + assert response3 is not None + assert type(response3) == bytes # noqa: E721 + assert response3 == response1 + + def test_exec_command__test_unset_dummy_var(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "TESTGRES_TEST__DUMMY_VAR_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) == bytes # noqa: E721 + assert response2 == b'\n' + + def test_is_executable_true(self, os_ops: OsOperations): + """ + Test is_executable for an existing executable. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + response = os_ops.is_executable("/bin/sh") + + assert response is True + + def test_is_executable_false(self, os_ops: OsOperations): + """ + Test is_executable for a non-executable. + """ + assert isinstance(os_ops, OsOperations) + + response = os_ops.is_executable(__file__) + + assert response is False + + def test_makedirs_and_rmdirs_success(self, os_ops: OsOperations): + """ + Test makedirs and rmdirs for successful directory creation and removal. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = "pwd" + pwd = os_ops.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() + + path = "{}/test_dir".format(pwd) + + # Test makedirs + os_ops.makedirs(path) + assert os.path.exists(path) + assert os_ops.path_exists(path) + + # Test rmdirs + os_ops.rmdirs(path) + assert not os.path.exists(path) + assert not os_ops.path_exists(path) + + def test_makedirs_failure(self, os_ops: OsOperations): + """ + Test makedirs for failure. + """ + # Try to create a directory in a read-only location + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/root/test_dir" + + # Test makedirs + with pytest.raises(Exception): + os_ops.makedirs(path) + + def test_listdir(self, os_ops: OsOperations): + """ + Test listdir for listing directory contents. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/etc" + files = os_ops.listdir(path) + assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) == str # noqa: E721 + + def test_path_exists_true__directory(self, os_ops: OsOperations): + """ + Test path_exists for an existing directory. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/etc") is True + + def test_path_exists_true__file(self, os_ops: OsOperations): + """ + Test path_exists for an existing file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists(__file__) is True + + def test_path_exists_false__directory(self, os_ops: OsOperations): + """ + Test path_exists for a non-existing directory. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/nonexistent_path") is False + + def test_path_exists_false__file(self, os_ops: OsOperations): + """ + Test path_exists for a non-existing file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/etc/nonexistent_path.txt") is False + + def test_mkdtemp__default(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkdtemp() + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_mkdtemp__custom(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_TEMPLATE = "abcdef" + path = os_ops.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + assert C_TEMPLATE in os.path.basename(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_rmdirs(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkdtemp() + assert os.path.exists(path) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + + def test_rmdirs__01_with_subfolder(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with subfolder + path = os_ops.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + os_ops.makedirs(dir1) + assert os.path.exists(dir1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + + def test_rmdirs__02_with_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with file + path = os_ops.mkdtemp() + assert os.path.exists(path) + + file1 = os.path.join(path, "file1.txt") + assert not os.path.exists(file1) + + os_ops.touch(file1) + assert os.path.exists(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(file1) + + def test_rmdirs__03_with_subfolder_and_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with subfolder and file + path = os_ops.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + os_ops.makedirs(dir1) + assert os.path.exists(dir1) + + file1 = os.path.join(dir1, "file1.txt") + assert not os.path.exists(file1) + + os_ops.touch(file1) + assert os.path.exists(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + assert not os.path.exists(file1) + + def test_write_text_file(self, os_ops: OsOperations): + """ + Test write for writing data to a text file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = os_ops.mkstemp() + data = "Hello, world!" + + os_ops.write(filename, data, truncate=True) + os_ops.write(filename, data) + + response = os_ops.read(filename) + + assert response == data + data + + os_ops.remove_file(filename) + + def test_write_binary_file(self, os_ops: OsOperations): + """ + Test write for writing data to a binary file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/tmp/test_file.bin" + data = b"\x00\x01\x02\x03" + + os_ops.write(filename, data, binary=True, truncate=True) + + response = os_ops.read(filename, binary=True) + + assert response == data + + def test_read_text_file(self, os_ops: OsOperations): + """ + Test read for reading data from a text file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/etc/hosts" + + response = os_ops.read(filename) + + assert isinstance(response, str) + + def test_read_binary_file(self, os_ops: OsOperations): + """ + Test read for reading data from a binary file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/usr/bin/python3" + + response = os_ops.read(filename, binary=True) + + assert isinstance(response, bytes) + + def test_read__text(self, os_ops: OsOperations): + """ + Test OsOperations::read for text data. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + with open(filename, 'r') as file: # open in a text mode + response0 = file.read() + + assert type(response0) == str # noqa: E721 + + response1 = os_ops.read(filename) + assert type(response1) == str # noqa: E721 + assert response1 == response0 + + response2 = os_ops.read(filename, encoding=None, binary=False) + assert type(response2) == str # noqa: E721 + assert response2 == response0 + + response3 = os_ops.read(filename, encoding="") + assert type(response3) == str # noqa: E721 + assert response3 == response0 + + response4 = os_ops.read(filename, encoding="UTF-8") + assert type(response4) == str # noqa: E721 + assert response4 == response0 + + def test_read__binary(self, os_ops: OsOperations): + """ + Test OsOperations::read for binary data. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = os_ops.read(filename, binary=True) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + def test_read__binary_and_encoding(self, os_ops: OsOperations): + """ + Test OsOperations::read for binary data and encoding. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + os_ops.read(filename, encoding="", binary=True) + + def test_read_binary__spec(self, os_ops: OsOperations): + """ + Test OsOperations::read_binary. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # currnt file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = os_ops.read_binary(filename, 0) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + response2 = os_ops.read_binary(filename, 1) + assert type(response2) == bytes # noqa: E721 + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = os_ops.read_binary(filename, len(response1)) + assert type(response3) == bytes # noqa: E721 + assert len(response3) == 0 + + response4 = os_ops.read_binary(filename, len(response2)) + assert type(response4) == bytes # noqa: E721 + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = os_ops.read_binary(filename, len(response1) + 1) + assert type(response5) == bytes # noqa: E721 + assert len(response5) == 0 + + def test_read_binary__spec__negative_offset(self, os_ops: OsOperations): + """ + Test OsOperations::read_binary with negative offset. + """ + assert isinstance(os_ops, OsOperations) + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + os_ops.read_binary(__file__, -1) + + def test_get_file_size(self, os_ops: OsOperations): + """ + Test OsOperations::get_file_size. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + sz0 = os.path.getsize(filename) + assert type(sz0) == int # noqa: E721 + + sz1 = os_ops.get_file_size(filename) + assert type(sz1) == int # noqa: E721 + assert sz1 == sz0 + + def test_isfile_true(self, os_ops: OsOperations): + """ + Test isfile for an existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ + + response = os_ops.isfile(filename) + + assert response is True + + def test_isfile_false__not_exist(self, os_ops: OsOperations): + """ + Test isfile for a non-existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") + + response = os_ops.isfile(filename) + + assert response is False + + def test_isfile_false__directory(self, os_ops: OsOperations): + """ + Test isfile for a firectory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.dirname(__file__) + + assert os_ops.isdir(name) + + response = os_ops.isfile(name) + + assert response is False + + def test_isdir_true(self, os_ops: OsOperations): + """ + Test isdir for an existing directory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.dirname(__file__) + + response = os_ops.isdir(name) + + assert response is True + + def test_isdir_false__not_exist(self, os_ops: OsOperations): + """ + Test isdir for a non-existing directory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") + + response = os_ops.isdir(name) + + assert response is False + + def test_isdir_false__file(self, os_ops: OsOperations): + """ + Test isdir for a file. + """ + assert isinstance(os_ops, OsOperations) + + name = __file__ + + assert os_ops.isfile(name) + + response = os_ops.isdir(name) + + assert response is False + + def test_cwd(self, os_ops: OsOperations): + """ + Test cwd. + """ + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + assert v != "" + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) == __class__.tagWriteData001 # noqa: E721 + return request.param + + def test_write(self, write_data001: tagWriteData001, os_ops: OsOperations): + assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 + assert isinstance(os_ops, OsOperations) + + mode = "w+b" if write_data001.call_param__binary else "w+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: + tmp_file.write(write_data001.source) + tmp_file.flush() + + os_ops.write( + tmp_file.name, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary) + + tmp_file.seek(0) + + s = tmp_file.read() + + assert s == write_data001.result + + def test_touch(self, os_ops: OsOperations): + """ + Test touch for creating a new file or updating access and modification times of an existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp() + + # TODO: this test does not check the result of 'touch' command! + + os_ops.touch(filename) + + assert os_ops.isfile(filename) + + os_ops.remove_file(filename) + + def test_is_port_free__true(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 128 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + r = os_ops.is_port_free(port) + + if r: + ok_count += 1 + logging.info("OK. Port {} is free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} is not free.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") + + def test_is_port_free__false(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 10 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + def LOCAL_server(s: socket.socket): + assert s is not None + assert type(s) == socket.socket # noqa: E721 + + try: + while True: + r = s.accept() + + if r is None: + break + except Exception as e: + assert e is not None + pass + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + th = threading.Thread(target=LOCAL_server, args=[s]) + + s.listen(10) + + assert type(th) == threading.Thread # noqa: E721 + th.start() + + try: + r = os_ops.is_port_free(port) + finally: + s.shutdown(2) + th.join() + + if not r: + ok_count += 1 + logging.info("OK. Port {} is not free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} does not accept connection.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") + + def test_get_tmpdir(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + dir = os_ops.get_tempdir() + assert type(dir) == str # noqa: E721 + assert os_ops.path_exists(dir) + assert os.path.exists(dir) + + file_path = os.path.join(dir, "testgres--" + uuid.uuid4().hex + ".tmp") + + os_ops.write(file_path, "1234", binary=False) + + assert os_ops.path_exists(file_path) + assert os.path.exists(file_path) + + d = os_ops.read(file_path, binary=False) + + assert d == "1234" + + os_ops.remove_file(file_path) + + assert not os_ops.path_exists(file_path) + assert not os.path.exists(file_path) + + def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + actual_dir = os_ops.get_tempdir() + assert actual_dir is not None + assert type(actual_dir) == str # noqa: E721 + expected_dir = str(tempfile.tempdir) + assert actual_dir == expected_dir + + class tagData_OS_OPS__NUMS: + os_ops_descr: OsOpsDescr + nums: int + + def __init__(self, os_ops_descr: OsOpsDescr, nums: int): + assert isinstance(os_ops_descr, OsOpsDescr) + assert type(nums) == int # noqa: E721 + + self.os_ops_descr = os_ops_descr + self.nums = nums + + sm_test_exclusive_creation__mt__data = [ + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000), + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120), + ] + + @pytest.fixture( + params=sm_test_exclusive_creation__mt__data, + ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS: + assert isinstance(request, pytest.FixtureRequest) + return request.param + + def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): + assert type(data001) == __class__.tagData_OS_OPS__NUMS # noqa: E721 + + N_WORKERS = 4 + N_NUMBERS = data001.nums + assert type(N_NUMBERS) == int # noqa: E721 + + os_ops = data001.os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + lock_dir_prefix = "test_mkdir_mt--" + uuid.uuid4().hex + + lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix) + + logging.info("A lock file [{}] is creating ...".format(lock_dir)) + + assert os.path.exists(lock_dir) + + def MAKE_PATH(lock_dir: str, num: int) -> str: + assert type(lock_dir) == str # noqa: E721 + assert type(num) == int # noqa: E721 + return os.path.join(lock_dir, str(num) + ".lock") + + def LOCAL_WORKER(os_ops: OsOperations, + workerID: int, + lock_dir: str, + cNumbers: int, + reservedNumbers: typing.Set[int]) -> None: + assert isinstance(os_ops, OsOperations) + assert type(workerID) == int # noqa: E721 + assert type(lock_dir) == str # noqa: E721 + assert type(cNumbers) == int # noqa: E721 + assert type(reservedNumbers) == set # noqa: E721 + assert cNumbers > 0 + assert len(reservedNumbers) == 0 + + assert os.path.exists(lock_dir) + + def LOG_INFO(template: str, *args: list) -> None: + assert type(template) == str # noqa: E721 + assert type(args) == tuple # noqa: E721 + + msg = template.format(*args) + assert type(msg) == str # noqa: E721 + + logging.info("[Worker #{}] {}".format(workerID, msg)) + return + + LOG_INFO("HELLO! I am here!") + + for num in range(cNumbers): + assert not (num in reservedNumbers) + + file_path = MAKE_PATH(lock_dir, num) + + try: + os_ops.makedir(file_path) + except Exception as e: + LOG_INFO( + "Can't reserve {}. Error ({}): {}", + num, + type(e).__name__, + str(e) + ) + continue + + LOG_INFO("Number {} is reserved!", num) + assert os_ops.path_exists(file_path) + reservedNumbers.add(num) + continue + + n_total = cNumbers + n_ok = len(reservedNumbers) + assert n_ok <= n_total + + LOG_INFO("Finish! OK: {}. FAILED: {}.", n_ok, n_total - n_ok) + return + + # ----------------------- + logging.info("Worker are creating ...") + + threadPool = ThreadPoolExecutor( + max_workers=N_WORKERS, + thread_name_prefix="ex_creator" + ) + + class tadWorkerData: + future: ThreadFuture + reservedNumbers: typing.Set[int] + + workerDatas: typing.List[tadWorkerData] = list() + + nErrors = 0 + + try: + for n in range(N_WORKERS): + logging.info("worker #{} is creating ...".format(n)) + + workerDatas.append(tadWorkerData()) + + workerDatas[n].reservedNumbers = set() + + workerDatas[n].future = threadPool.submit( + LOCAL_WORKER, + os_ops, + n, + lock_dir, + N_NUMBERS, + workerDatas[n].reservedNumbers + ) + + assert workerDatas[n].future is not None + + logging.info("OK. All the workers were created!") + except Exception as e: + nErrors += 1 + logging.error("A problem is detected ({}): {}".format(type(e).__name__, str(e))) + + logging.info("Will wait for stop of all the workers...") + + nWorkers = 0 + + assert type(workerDatas) == list # noqa: E721 + + for i in range(len(workerDatas)): + worker = workerDatas[i].future + + if worker is None: + continue + + nWorkers += 1 + + assert isinstance(worker, ThreadFuture) + + try: + logging.info("Wait for worker #{}".format(i)) + worker.result() + except Exception as e: + nErrors += 1 + logging.error("Worker #{} finished with error ({}): {}".format( + i, + type(e).__name__, + str(e), + )) + continue + + assert nWorkers == N_WORKERS + + if nErrors != 0: + raise RuntimeError("Some problems were detected. Please examine the log messages.") + + logging.info("OK. Let's check worker results!") + + reservedNumbers: typing.Dict[int, int] = dict() + + for i in range(N_WORKERS): + logging.info("Worker #{} is checked ...".format(i)) + + workerNumbers = workerDatas[i].reservedNumbers + assert type(workerNumbers) == set # noqa: E721 + + for n in workerNumbers: + if n < 0 or n >= N_NUMBERS: + nErrors += 1 + logging.error("Unexpected number {}".format(n)) + continue + + if n in reservedNumbers.keys(): + nErrors += 1 + logging.error("Number {} was already reserved by worker #{}".format( + n, + reservedNumbers[n] + )) + else: + reservedNumbers[n] = i + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + continue + + logging.info("OK. Let's check reservedNumbers!") + + for n in range(N_NUMBERS): + if not (n in reservedNumbers.keys()): + nErrors += 1 + logging.error("Number {} is not reserved!".format(n)) + continue + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + # OK! + continue + + logging.info("Verification is finished! Total error count is {}.".format(nErrors)) + + if nErrors == 0: + logging.info("Root lock-directory [{}] will be deleted.".format( + lock_dir + )) + + for n in range(N_NUMBERS): + file_path = MAKE_PATH(lock_dir, n) + try: + os_ops.rmdir(file_path) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + file_path, + type(e).__name__, + str(e) + )) + continue + + if os_ops.path_exists(file_path): + nErrors += 1 + logging.error("Directory {} is not deleted!".format(file_path)) + continue + + if nErrors == 0: + try: + os_ops.rmdir(lock_dir) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + lock_dir, + type(e).__name__, + str(e) + )) + + logging.info("Test is finished! Total error count is {}.".format(nErrors)) + return diff --git a/tests/test_os_ops_local.py b/tests/test_os_ops_local.py new file mode 100644 index 00000000..f60c3fc9 --- /dev/null +++ b/tests/test_os_ops_local.py @@ -0,0 +1,60 @@ +# coding: utf-8 +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations + +import os + +import pytest +import re + + +class TestOsOpsLocal: + @pytest.fixture + def os_ops(self): + return OsOpsDescrs.sm_local_os_ops + + def test_read__unknown_file(self, os_ops: OsOperations): + """ + Test LocalOperations::read with unknown file. + """ + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + os_ops.read("/dummy") + + def test_read_binary__spec__unk_file(self, os_ops: OsOperations): + """ + Test LocalOperations::read_binary with unknown file. + """ + + with pytest.raises( + FileNotFoundError, + match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + os_ops.read_binary("/dummy", 0) + + def test_get_file_size__unk_file(self, os_ops: OsOperations): + """ + Test LocalOperations::get_file_size. + """ + assert isinstance(os_ops, OsOperations) + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + os_ops.get_file_size("/dummy") + + def test_cwd(self, os_ops: OsOperations): + """ + Test cwd. + """ + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + + expectedValue = os.getcwd() + assert expectedValue is not None + assert type(expectedValue) == str # noqa: E721 + assert expectedValue != "" # research + + # Comp result + assert v == expectedValue diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py new file mode 100755 index 00000000..65830218 --- /dev/null +++ b/tests/test_os_ops_remote.py @@ -0,0 +1,79 @@ +# coding: utf-8 + +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations + +from testgres import ExecUtilException + +import os +import pytest + + +class TestOsOpsRemote: + @pytest.fixture + def os_ops(self): + return OsOpsDescrs.sm_remote_os_ops + + def test_rmdirs__try_to_delete_nonexist_path(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = "/root/test_dir" + + assert os_ops.rmdirs(path, ignore_errors=False) is True + + def test_rmdirs__try_to_delete_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkstemp() + assert type(path) == str # noqa: E721 + assert os.path.exists(path) + + with pytest.raises(ExecUtilException) as x: + os_ops.rmdirs(path, ignore_errors=False) + + assert os.path.exists(path) + assert type(x.value) == ExecUtilException # noqa: E721 + assert x.value.message == "Utility exited with non-zero code (20). Error: `cannot remove '" + path + "': it is not a directory`" + assert type(x.value.error) == str # noqa: E721 + assert x.value.error.strip() == "cannot remove '" + path + "': it is not a directory" + assert type(x.value.exit_code) == int # noqa: E721 + assert x.value.exit_code == 20 + + def test_read__unknown_file(self, os_ops: OsOperations): + """ + Test RemoteOperations::read with unknown file. + """ + assert isinstance(os_ops, OsOperations) + + with pytest.raises(ExecUtilException) as x: + os_ops.read("/dummy") + + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + + def test_read_binary__spec__unk_file(self, os_ops: OsOperations): + """ + Test RemoteOperations::read_binary with unknown file. + """ + assert isinstance(os_ops, OsOperations) + + with pytest.raises(ExecUtilException) as x: + os_ops.read_binary("/dummy", 0) + + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + + def test_get_file_size__unk_file(self, os_ops: OsOperations): + """ + Test RemoteOperations::get_file_size. + """ + assert isinstance(os_ops, OsOperations) + + with pytest.raises(ExecUtilException) as x: + os_ops.get_file_size("/dummy") + + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) diff --git a/tests/test_simple.py b/tests/test_simple.py deleted file mode 100755 index 79e0c913..00000000 --- a/tests/test_simple.py +++ /dev/null @@ -1,652 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -import subprocess -import tempfile -import testgres -import time -import unittest - -import logging.config - -from distutils.version import LooseVersion - -from testgres import \ - InitNodeException, \ - StartNodeException, \ - ExecUtilException, \ - BackupException, \ - QueryException, \ - CatchUpException, \ - TimeoutException - -from testgres import \ - TestgresConfig, \ - configure_testgres - -from testgres import \ - NodeStatus, \ - IsolationLevel, \ - get_new_node - -from testgres import \ - get_bin_path, \ - get_pg_config - -from testgres import bound_ports - - -def util_is_executable(util): - def good_properties(f): - return ( - os.path.exists(f) and - os.path.isfile(f) and - os.access(f, os.X_OK) - ) - - # try to resolve it - if good_properties(get_bin_path(util)): - return True - - # check if util is in PATH - for path in os.environ["PATH"].split(os.pathsep): - if good_properties(os.path.join(path, util)): - return True - - -class SimpleTest(unittest.TestCase): - def test_custom_init(self): - with get_new_node() as node: - # enable page checksums - node.init(initdb_params=['-k']).start() - node.safe_psql('select 1') - - with get_new_node() as node: - node.init( - allow_streaming=True, - initdb_params=['--auth-local=reject', '--auth-host=reject']) - - hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - with open(hba_file, 'r') as conf: - lines = conf.readlines() - - # check number of lines - self.assertGreaterEqual(len(lines), 6) - - # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) - - def test_double_init(self): - with get_new_node().init() as node: - - # can't initialize node more than once - with self.assertRaises(InitNodeException): - node.init() - - def test_init_after_cleanup(self): - with get_new_node() as node: - node.init().start() - node.status() - node.safe_psql('select 1') - - node.cleanup() - - node.init().start() - node.status() - node.safe_psql('select 1') - - def test_double_start(self): - with get_new_node() as node: - node.init().start() - - # can't start node more than once - with self.assertRaises(StartNodeException): - node.start() - - def test_uninitialized_start(self): - with get_new_node() as node: - # node is not initialized yet - with self.assertRaises(StartNodeException): - node.start() - - def test_restart(self): - with get_new_node() as node: - node.init().start() - - # restart, ok - res = node.execute('select 1') - self.assertEqual(res, [(1, )]) - node.restart() - res = node.execute('select 2') - self.assertEqual(res, [(2, )]) - - # restart, fail - with self.assertRaises(StartNodeException): - node.append_conf('pg_hba.conf', 'DUMMY') - node.restart() - - def test_reload(self): - with get_new_node() as node: - node.init().start() - - # change client_min_messages and save old value - cmm_old = node.execute('show client_min_messages') - node.append_conf('postgresql.conf', 'client_min_messages = DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) - - def test_pg_ctl(self): - with get_new_node() as node: - node.init().start() - - status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) - - def test_status(self): - # check NodeStatus cast to bool - self.assertTrue(NodeStatus.Running) - - # check NodeStatus cast to bool - self.assertFalse(NodeStatus.Stopped) - - # check NodeStatus cast to bool - self.assertFalse(NodeStatus.Uninitialized) - - # check statuses after each operation - with get_new_node() as node: - self.assertEqual(node.get_pid(), 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) - - node.init() - - self.assertEqual(node.get_pid(), 0) - self.assertEqual(node.status(), NodeStatus.Stopped) - - node.start() - - self.assertTrue(node.get_pid() > 0) - self.assertEqual(node.status(), NodeStatus.Running) - - node.stop() - - self.assertEqual(node.get_pid(), 0) - self.assertEqual(node.status(), NodeStatus.Stopped) - - node.cleanup() - - self.assertEqual(node.get_pid(), 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) - - def test_psql(self): - with get_new_node().init().start() as node: - - # check returned values (1 arg) - res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) - - # check returned values (named) - res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') - - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') - - # check returned values (named) - res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') - - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql( - 'copy horns from stdin (format csv)', input=b"1\n2\n3\n\.\n") - _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') - - # check psql's default args, fails - with self.assertRaises(QueryException): - node.psql() - - node.stop() - - # check psql on stopped node, fails - with self.assertRaises(QueryException): - node.safe_psql('select 1') - - def test_transactions(self): - with get_new_node().init().start() as node: - - with node.connect() as con: - con.begin() - con.execute('create table test(val int)') - con.execute('insert into test values (1)') - con.commit() - - con.begin() - con.execute('insert into test values (2)') - res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1, ), (2, )]) - con.rollback() - - con.begin() - res = con.execute('select * from test') - self.assertListEqual(res, [(1, )]) - con.rollback() - - con.begin() - con.execute('drop table test') - con.commit() - - def test_control_data(self): - with get_new_node() as node: - - # node is not initialized yet - with self.assertRaises(ExecUtilException): - node.get_control_data() - - node.init() - data = node.get_control_data() - - # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) - - def test_backup_simple(self): - with get_new_node() as master: - - # enable streaming for backups - master.init(allow_streaming=True) - - # node must be running - with self.assertRaises(BackupException): - master.backup() - - # it's time to start node - master.start() - - # fill node with some data - master.psql('create table test as select generate_series(1, 4) i') - - with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary() as slave: - slave.start() - res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) - - def test_backup_multiple(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup1, \ - node.backup(xlog_method='fetch') as backup2: - - self.assertNotEqual(backup1.base_dir, backup2.base_dir) - - with node.backup(xlog_method='fetch') as backup: - with backup.spawn_primary('node1', destroy=False) as node1, \ - backup.spawn_primary('node2', destroy=False) as node2: - - self.assertNotEqual(node1.base_dir, node2.base_dir) - - def test_backup_exhaust(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup: - - # exhaust backup by creating new node - with backup.spawn_primary() as node1: # noqa - pass - - # now let's try to create one more node - with self.assertRaises(BackupException): - with backup.spawn_primary() as node2: # noqa - pass - - def test_backup_and_replication(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - node.psql('create table abc(a int, b int)') - node.psql('insert into abc values (1, 2)') - - backup = node.backup() - - with backup.spawn_replica().start() as replica: - res = replica.execute('select * from abc') - self.assertListEqual(res, [(1, 2)]) - - # Insert into master node - node.psql('insert into abc values (3, 4)') - - # Wait until data syncronizes - replica.catchup() - - # Check that this record was exported to replica - res = replica.execute('select * from abc') - self.assertListEqual(res, [(1, 2), (3, 4)]) - - def test_replicate(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate().start() as replica: - res = replica.execute('select 1') - self.assertListEqual(res, [(1, )]) - - node.execute('create table test (val int)', commit=True) - - replica.catchup() - - res = node.execute('select * from test') - self.assertListEqual(res, []) - - def test_incorrect_catchup(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - # node has no master, can't catch up - with self.assertRaises(CatchUpException): - node.catchup() - - def test_dump(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - - # take a new dump - dump = node1.dump() - self.assertTrue(os.path.isfile(dump)) - - with get_new_node().init().start() as node2: - - # restore dump - node2.restore(filename=dump) - - res = node2.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - - # finally, remove dump - os.remove(dump) - - def test_users(self): - with get_new_node().init().start() as node: - node.psql('create role test_user login') - value = node.safe_psql('select 1', username='test_user') - self.assertEqual(value, b'1\n') - - def test_poll_query_until(self): - with get_new_node() as node: - node.init().start() - - get_time = 'select extract(epoch from now())' - check_time = 'select extract(epoch from now()) - {} >= 5' - - start_time = node.execute(get_time)[0][0] - node.poll_query_until(query=check_time.format(start_time)) - end_time = node.execute(get_time)[0][0] - - self.assertTrue(end_time - start_time >= 5) - - # check 0 rows - with self.assertRaises(QueryException): - node.poll_query_until( - query='select * from pg_class where true = false') - - # check 0 columns - with self.assertRaises(QueryException): - node.poll_query_until(query='select from pg_class limit 1') - - # check None, fail - with self.assertRaises(QueryException): - node.poll_query_until(query='create table abc (val int)') - - # check None, ok - node.poll_query_until( - query='create table def()', expected=None) # returns nothing - - # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): - node.poll_query_until( - query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) - - # check arbitrary expected value, ok - node.poll_query_until(query='select 2', expected=2) - - # check timeout - with self.assertRaises(TimeoutException): - node.poll_query_until( - query='select 1 > 2', max_attempts=3, sleep_time=0.01) - - # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): - node.poll_query_until(query='dummy1') - - # check ProgrammingError, ok - with self.assertRaises(TimeoutException): - node.poll_query_until( - query='dummy2', - max_attempts=3, - sleep_time=0.01, - raise_programming_error=False) - - # check 1 arg, ok - node.poll_query_until('select true') - - def test_logging(self): - logfile = tempfile.NamedTemporaryFile('w', delete=True) - - log_conf = { - 'version': 1, - 'handlers': { - 'file': { - 'class': 'logging.FileHandler', - 'filename': logfile.name, - 'formatter': 'base_format', - 'level': logging.DEBUG, - }, - }, - 'formatters': { - 'base_format': { - 'format': '%(node)-5s: %(message)s', - }, - }, - 'root': { - 'handlers': ('file', ), - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(log_conf) - - node_name = 'master' - with get_new_node(name=node_name, use_logging=True) as master: - master.init().start() - - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) - - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - self.assertTrue(master._logger.is_alive()) - - @unittest.skipUnless( - util_is_executable("pgbench"), "pgbench may be missing") - def test_pgbench(self): - with get_new_node().init().start() as node: - - # initialize pgbench DB and run benchmarks - node.pgbench_init( - scale=2, foreign_keys=True, options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - - out, _ = proc.communicate() - out = out.decode('utf-8') - - self.assertTrue('tps' in out) - - def test_pg_config(self): - # set global if it wasn't set - configure_testgres(cache_pg_config=True) - - # check same instances - a = get_pg_config() - b = get_pg_config() - self.assertEqual(id(a), id(b)) - - # save right before config change - c1 = get_pg_config() - - # modify setting - configure_testgres(cache_pg_config=False) - self.assertFalse(TestgresConfig.cache_pg_config) - - # save right after config change - c2 = get_pg_config() - - # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) - - # check different instances - a = get_pg_config() - b = get_pg_config() - self.assertNotEqual(id(a), id(b)) - - # restore setting - configure_testgres(cache_pg_config=True) - - def test_unix_sockets(self): - with get_new_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - node.execute('select 1') - node.safe_psql('select 1') - - with node.replicate().start() as r: - r.execute('select 1') - r.safe_psql('select 1') - - def test_auto_name(self): - with get_new_node().init(allow_streaming=True).start() as m: - with m.replicate().start() as r: - - # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) - - # check their names - self.assertIsNotNone(m.name) - self.assertIsNotNone(r.name) - self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) - - def test_file_tail(self): - from testgres.utils import file_tail - - s1 = "the quick brown fox jumped over that lazy dog\n" - s2 = "abc\n" - s3 = "def\n" - - with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - for i in range(1, 5000): - f.write(s1) - f.write(s2) - f.write(s3) - - f.seek(0) - lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) - - f.seek(0) - lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) - - def test_isolation_levels(self): - with get_new_node().init().start() as node: - with node.connect() as con: - # string levels - con.begin('Read Uncommitted').commit() - con.begin('Read Committed').commit() - con.begin('Repeatable Read').commit() - con.begin('Serializable').commit() - - # enum levels - con.begin(IsolationLevel.ReadUncommitted).commit() - con.begin(IsolationLevel.ReadCommitted).commit() - con.begin(IsolationLevel.RepeatableRead).commit() - con.begin(IsolationLevel.Serializable).commit() - - # check wrong level - with self.assertRaises(QueryException): - con.begin('Garbage').commit() - - def test_ports_management(self): - # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) - - with get_new_node() as node: - # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) - - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - self.assertEqual(port_1, port_2) - - # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) - - def test_version_management(self): - a = LooseVersion('10.0') - b = LooseVersion('10') - c = LooseVersion('9.6.5') - - self.assertTrue(a > b) - self.assertTrue(b > c) - self.assertTrue(a > c) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py new file mode 100644 index 00000000..cf203a67 --- /dev/null +++ b/tests/test_testgres_common.py @@ -0,0 +1,1590 @@ +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices +from .helpers.global_data import OsOperations +from .helpers.global_data import PortManager + +from testgres.node import PgVer +from testgres.node import PostgresNode +from testgres.node import PostgresNodeLogReader +from testgres.node import PostgresNodeUtils +from testgres.utils import get_pg_version2 +from testgres.utils import file_tail +from testgres.utils import get_bin_path2 +from testgres import ProcessType +from testgres import NodeStatus +from testgres import IsolationLevel + +# New name prevents to collect test-functions in TestgresException and fixes +# the problem with pytest warning. +from testgres import TestgresException as testgres_TestgresException + +from testgres import InitNodeException +from testgres import StartNodeException +from testgres import QueryException +from testgres import ExecUtilException +from testgres import TimeoutException +from testgres import InvalidOperationException +from testgres import BackupException +from testgres import ProgrammingError +from testgres import scoped_config +from testgres import First, Any + +from contextlib import contextmanager + +import pytest +import six +import logging +import time +import tempfile +import uuid +import os +import re +import subprocess +import typing + + +@contextmanager +def removing(os_ops: OsOperations, f): + assert isinstance(os_ops, OsOperations) + + try: + yield f + finally: + if os_ops.isfile(f): + os_ops.remove_file(f) + + elif os_ops.isdir(f): + os_ops.rmdirs(f, ignore_errors=True) + + +class TestTestgresCommon: + sm_node_svcs: typing.List[PostgresNodeService] = [ + PostgresNodeServices.sm_local, + PostgresNodeServices.sm_local2, + PostgresNodeServices.sm_remote, + ] + + @pytest.fixture( + params=sm_node_svcs, + ids=[descr.sign for descr in sm_node_svcs] + ) + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) + return request.param + + def test_version_management(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + a = PgVer('10.0') + b = PgVer('10') + c = PgVer('9.6.5') + d = PgVer('15.0') + e = PgVer('15rc1') + f = PgVer('15beta4') + h = PgVer('15.3biha') + i = PgVer('15.3') + g = PgVer('15.3.1bihabeta1') + k = PgVer('15.3.1') + + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) + + version = get_pg_version2(node_svc.os_ops) + + with __class__.helper__get_node(node_svc) as node: + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) + + def test_node_repr(self, node_svc: PostgresNodeService): + with __class__.helper__get_node(node_svc).init() as node: + pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" + assert re.match(pattern, str(node)) is not None + + def test_custom_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + # enable page checksums + node.init(initdb_params=['-k']).start() + + with __class__.helper__get_node(node_svc) as node: + node.init( + allow_streaming=True, + initdb_params=['--auth-local=reject', '--auth-host=reject']) + + hba_file = os.path.join(node.data_dir, 'pg_hba.conf') + lines = node.os_ops.readlines(hba_file) + + # check number of lines + assert (len(lines) >= 6) + + # there should be no trust entries at all + assert not (any('trust' in s for s in lines)) + + def test_double_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc).init() as node: + # can't initialize node more than once + with pytest.raises(expected_exception=InitNodeException): + node.init() + + def test_init_after_cleanup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start().execute('select 1') + node.cleanup() + node.init().start().execute('select 1') + + def test_init_unique_system_id(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + # this function exists in PostgreSQL 9.6+ + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') + + query = 'select system_identifier from pg_control_system()' + + with scoped_config(cache_initdb=False): + with __class__.helper__get_node(node_svc).init().start() as node0: + id0 = node0.execute(query)[0] + + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: + assert (config.cache_initdb) + assert (config.cached_initdb_unique) + + # spawn two nodes; ids must be different + with __class__.helper__get_node(node_svc).init().start() as node1, \ + __class__.helper__get_node(node_svc).init().start() as node2: + id1 = node1.execute(query)[0] + id2 = node2.execute(query)[0] + + # ids must increase + assert (id1 > id0) + assert (id2 > id1) + + def test_node_exit(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with pytest.raises(expected_exception=QueryException): + with __class__.helper__get_node(node_svc).init() as node: + base_dir = node.base_dir + node.safe_psql('select 1') + + # we should save the DB for "debugging" + assert (node_svc.os_ops.path_exists(base_dir)) + node_svc.os_ops.rmdirs(base_dir, ignore_errors=True) + + with __class__.helper__get_node(node_svc).init() as node: + base_dir = node.base_dir + + # should have been removed by default + assert not (node_svc.os_ops.path_exists(base_dir)) + + def test_double_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc).init().start() as node: + # can't start node more than once + node.start() + assert (node.is_started) + + def test_uninitialized_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + # node is not initialized yet + with pytest.raises(expected_exception=StartNodeException): + node.start() + + def test_restart(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + + # restart, ok + res = node.execute('select 1') + assert (res == [(1,)]) + node.restart() + res = node.execute('select 2') + assert (res == [(2,)]) + + # restart, fail + with pytest.raises(expected_exception=StartNodeException): + node.append_conf('pg_hba.conf', 'DUMMY') + node.restart() + + def test_reload(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + + # change client_min_messages and save old value + cmm_old = node.execute('show client_min_messages') + node.append_conf(client_min_messages='DEBUG1') + + # reload config + node.reload() + + # check new value + cmm_new = node.execute('show client_min_messages') + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) + + def test_pg_ctl(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + + status = node.pg_ctl(['status']) + assert ('PID' in status) + + def test_status(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) + + # check statuses after each operation + with __class__.helper__get_node(node_svc) as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.start() + + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) + + node.stop() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.cleanup() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + def test_child_pids(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + master_processes = [ + ProcessType.AutovacuumLauncher, + ProcessType.BackgroundWriter, + ProcessType.Checkpointer, + ProcessType.StatsCollector, + ProcessType.WalSender, + ProcessType.WalWriter, + ] + + postgresVersion = get_pg_version2(node_svc.os_ops) + + if __class__.helper__pg_version_ge(postgresVersion, '10'): + master_processes.append(ProcessType.LogicalReplicationLauncher) + + if __class__.helper__pg_version_ge(postgresVersion, '14'): + master_processes.remove(ProcessType.StatsCollector) + + repl_processes = [ + ProcessType.Startup, + ProcessType.WalReceiver, + ] + + def LOCAL__test_auxiliary_pids( + node: PostgresNode, + expectedTypes: typing.List[ProcessType] + ) -> typing.List[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result: typing.List[ProcessType] = list() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: PostgresNode, + expectedTypes: typing.List[ProcessType]): + assert node is not None + assert type(node) == PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + + with __class__.helper__get_node(node_svc).init().start() as master: + + # master node doesn't have a source walsender! + with pytest.raises(expected_exception=testgres_TestgresException): + master.source_walsender + + with master.connect() as con: + assert (con.pid > 0) + + with master.replicate().start() as replica: + assert type(replica) == PostgresNode # noqa: E721 + + # test __str__ method + str(master.child_processes[0]) + + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) + + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids + + # there should be exactly 1 source walsender for replica + assert (len(master_pids[ProcessType.WalSender]) == 1) + pid1 = master_pids[ProcessType.WalSender][0] + pid2 = replica.source_walsender.pid + assert (pid1 == pid2) + + replica.stop() + + # there should be no walsender after we've stopped replica + with pytest.raises(expected_exception=testgres_TestgresException): + replica.source_walsender + + def test_exceptions(self): + str(StartNodeException('msg', [('file', 'lines')])) + str(ExecUtilException('msg', 'cmd', 1, 'out')) + str(QueryException('msg', 'query')) + + def test_auto_name(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc).init(allow_streaming=True).start() as m: + with m.replicate().start() as r: + # check that nodes are running + assert (m.status()) + assert (r.status()) + + # check their names + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) + + def test_file_tail(self): + s1 = "the quick brown fox jumped over that lazy dog\n" + s2 = "abc\n" + s3 = "def\n" + + with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: + sz = 0 + while sz < 3 * 8192: + sz += len(s1) + f.write(s1) + f.write(s2) + f.write(s3) + + f.seek(0) + lines = file_tail(f, 3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) + + f.seek(0) + lines = file_tail(f, 1) + assert (lines[0] == s3) + + def test_isolation_levels(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + with node.connect() as con: + # string levels + con.begin('Read Uncommitted').commit() + con.begin('Read Committed').commit() + con.begin('Repeatable Read').commit() + con.begin('Serializable').commit() + + # enum levels + con.begin(IsolationLevel.ReadUncommitted).commit() + con.begin(IsolationLevel.ReadCommitted).commit() + con.begin(IsolationLevel.RepeatableRead).commit() + con.begin(IsolationLevel.Serializable).commit() + + # check wrong level + with pytest.raises(expected_exception=QueryException): + con.begin('Garbage').commit() + + def test_users(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + node.psql('create role test_user login') + value = node.safe_psql('select 1', username='test_user') + value = __class__.helper__rm_carriage_returns(value) + assert (value == b'1\n') + + def test_poll_query_until(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init().start() + + get_time = 'select extract(epoch from now())' + check_time = 'select extract(epoch from now()) - {} >= 5' + + start_time = node.execute(get_time)[0][0] + node.poll_query_until(query=check_time.format(start_time)) + end_time = node.execute(get_time)[0][0] + + assert (end_time - start_time >= 5) + + # check 0 columns + with pytest.raises(expected_exception=QueryException): + node.poll_query_until( + query='select from pg_catalog.pg_class limit 1') + + # check None, fail + with pytest.raises(expected_exception=QueryException): + node.poll_query_until(query='create table abc (val int)') + + # check None, ok + node.poll_query_until(query='create table def()', + expected=None) # returns nothing + + # check 0 rows equivalent to expected=None + node.poll_query_until( + query='select * from pg_catalog.pg_class where true = false', + expected=None) + + # check arbitrary expected value, fail + with pytest.raises(expected_exception=TimeoutException): + node.poll_query_until(query='select 3', + expected=1, + max_attempts=3, + sleep_time=0.01) + + # check arbitrary expected value, ok + node.poll_query_until(query='select 2', expected=2) + + # check timeout + with pytest.raises(expected_exception=TimeoutException): + node.poll_query_until(query='select 1 > 2', + max_attempts=3, + sleep_time=0.01) + + # check ProgrammingError, fail + with pytest.raises(expected_exception=ProgrammingError): + node.poll_query_until(query='dummy1') + + # check ProgrammingError, ok + with pytest.raises(expected_exception=(TimeoutException)): + node.poll_query_until(query='dummy2', + max_attempts=3, + sleep_time=0.01, + suppress={ProgrammingError}) + + # check 1 arg, ok + node.poll_query_until('select true') + + def test_logging(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with __class__.helper__get_node(node_svc, name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) == list # noqa: E721 + + def LOCAL__test_lines(): + for s in lines: + if any(C_NODE_NAME in s for s in lines): + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + logging._acquireLock() + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) + logging._releaseLock() + # GO HOME! + return + + def test_psql(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + + # check returned values (1 arg) + res = node.psql('select 1') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + # check returned values (2 args) + res = node.psql('postgres', 'select 2') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'2\n', b'')) + + # check returned values (named) + res = node.psql(query='select 3', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'3\n', b'')) + + # check returned values (1 arg) + res = node.safe_psql('select 4') + assert (__class__.helper__rm_carriage_returns(res) == b'4\n') + + # check returned values (2 args) + res = node.safe_psql('postgres', 'select 5') + assert (__class__.helper__rm_carriage_returns(res) == b'5\n') + + # check returned values (named) + res = node.safe_psql(query='select 6', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == b'6\n') + + # check feeding input + node.safe_psql('create table horns (w int)') + node.safe_psql('copy horns from stdin (format csv)', + input=b"1\n2\n3\n\\.\n") + _sum = node.safe_psql('select sum(w) from horns') + assert (__class__.helper__rm_carriage_returns(_sum) == b'6\n') + + # check psql's default args, fails + with pytest.raises(expected_exception=QueryException): + r = node.psql() # raises! + logging.error("node.psql returns [{}]".format(r)) + + node.stop() + + # check psql on stopped node, fails + with pytest.raises(expected_exception=QueryException): + # [2025-04-03] This call does not raise exception! I do not know why. + r = node.safe_psql('select 1') # raises! + logging.error("node.safe_psql returns [{}]".format(r)) + + def test_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + def test_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + res = node.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + res2 = __class__.helper__rm_carriage_returns(res) + + assert res2[0] != 0 + assert b"DUMMY_HOST_NAME" in res[2] + + def test_safe_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + + with pytest.raises(expected_exception=Exception) as x: + node.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + assert "DUMMY_HOST_NAME" in str(x.value) + + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + assert (type(err) == str) # noqa: E721 + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) + + # --------- + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): + node.safe_psql("select 1;", expect_error=True) + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_transactions(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + + with node.connect() as con: + con.begin() + con.execute('create table test(val int)') + con.execute('insert into test values (1)') + con.commit() + + con.begin() + con.execute('insert into test values (2)') + res = con.execute('select * from test order by val asc') + assert (res == [(1, ), (2, )]) + con.rollback() + + con.begin() + res = con.execute('select * from test') + assert (res == [(1, )]) + con.rollback() + + con.begin() + con.execute('drop table test') + con.commit() + + def test_control_data(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + + # node is not initialized yet + with pytest.raises(expected_exception=ExecUtilException): + node.get_control_data() + + node.init() + data = node.get_control_data() + + # check returned dict + assert data is not None + assert (any('pg_control' in s for s in data.keys())) + + def test_backup_simple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: + + # enable streaming for backups + master.init(allow_streaming=True) + + # node must be running + with pytest.raises(expected_exception=BackupException): + master.backup() + + # it's time to start node + master.start() + + # fill node with some data + master.psql('create table test as select generate_series(1, 4) i') + + with master.backup(xlog_method='stream') as backup: + with backup.spawn_primary().start() as slave: + res = slave.execute('select * from test order by i asc') + assert (res == [(1, ), (2, ), (3, ), (4, )]) + + def test_backup_multiple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup1, \ + node.backup(xlog_method='fetch') as backup2: + assert (backup1.base_dir != backup2.base_dir) + + with node.backup(xlog_method='fetch') as backup: + with backup.spawn_primary('node1', destroy=False) as node1, \ + backup.spawn_primary('node2', destroy=False) as node2: + assert (node1.base_dir != node2.base_dir) + + def test_backup_exhaust(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup: + # exhaust backup by creating new node + with backup.spawn_primary(): + pass + + # now let's try to create one more node + with pytest.raises(expected_exception=BackupException): + backup.spawn_primary() + + def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): + node.backup(xlog_method='wrong') + + def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + C_MAX_ATTEMPT = 5 + + nAttempt = 0 + + while True: + if nAttempt == C_MAX_ATTEMPT: + raise Exception("PostgresSQL did not start.") + + nAttempt += 1 + logging.info("------------------------ NODE #{}".format( + nAttempt + )) + + with __class__.helper__get_node(node_svc, port=12345) as node: + if self.impl__test_pg_ctl_wait_option(node_svc, node): + break + continue + + logging.info("OK. Test is passed. Number of attempts is {}".format( + nAttempt + )) + return + + def impl__test_pg_ctl_wait_option( + self, + node_svc: PostgresNodeService, + node: PostgresNode + ) -> None: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node, PostgresNode) + assert node.status() == NodeStatus.Uninitialized + + C_MAX_ATTEMPTS = 50 + + node.init() + assert node.status() == NodeStatus.Stopped + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=True) + + node.start(wait=False) + nAttempt = 0 + while True: + if PostgresNodeUtils.delect_port_conflict(node_log_reader): + logging.info("Node port {} conflicted with another PostgreSQL instance.".format( + node.port + )) + return False + + if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + if s1 == NodeStatus.Running: + continue + + if s1 == NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + return True + + def test_replicate(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.replicate().start() as replica: + res = replica.execute('select 1') + assert (res == [(1, )]) + + node.execute('create table test (val int)', commit=True) + + replica.catchup() + + res = node.execute('select * from test') + assert (res == []) + + def test_synchronous_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") + + with __class__.helper__get_node(node_svc) as master: + old_version = not __class__.helper__pg_version_ge(current_version, '9.6') + + master.init(allow_streaming=True).start() + + if not old_version: + master.append_conf('synchronous_commit = remote_apply') + + # create standby + with master.replicate() as standby1, master.replicate() as standby2: + standby1.start() + standby2.start() + + # check formatting + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable + + # set synchronous_standby_names + master.set_synchronous_standbys(First(2, [standby1, standby2])) + master.restart() + + # the following part of the test is only applicable to newer + # versions of PostgresQL + if not old_version: + master.safe_psql('create table abc(a int)') + + # Create a large transaction that will take some time to apply + # on standby to check that it applies synchronously + # (If set synchronous_commit to 'on' or other lower level then + # standby most likely won't catchup so fast and test will fail) + master.safe_psql( + 'insert into abc select generate_series(1, 1000000)') + res = standby1.safe_psql('select count(*) from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') + + def test_logical_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (a int, b int)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + + # create publication / create subscription + pub = node1.publish('mypub') + sub = node2.subscribe(pub, 'mysub') + + node1.safe_psql('insert into test values (1, 1), (2, 2)') + + # wait until changes apply on subscriber and check them + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2)]) + + # disable and put some new data + sub.disable() + node1.safe_psql('insert into test values (3, 3)') + + # enable and ensure that data successfully transferred + sub.enable() + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3)]) + + # Add new tables. Since we added "all tables" to publication + # (default behaviour of publish() method) we don't need + # to explicitly perform pub.add_tables() + create_table = 'create table test2 (c char)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + sub.refresh() + + # put new data + node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + # drop subscription + sub.drop() + pub.drop() + + # create new publication and subscription for specific table + # (omitting copying data as it's already done) + pub = node1.publish('newpub', tables=['test']) + sub = node2.subscribe(pub, 'newsub', copy_data=False) + + node1.safe_psql('insert into test values (4, 4)') + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) + + # explicitly add table + with pytest.raises(expected_exception=ValueError): + pub.add_tables([]) # fail + pub.add_tables(['test2']) + node1.safe_psql('insert into test2 values (\'c\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + def test_logical_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + """ Runs catchup for 100 times to be sure that it is consistent """ + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (key int primary key, val int); ' + node1.safe_psql(create_table) + node1.safe_psql('alter table test replica identity default') + node2.safe_psql(create_table) + + # create publication / create subscription + sub = node2.subscribe(node1.publish('mypub'), 'mysub') + + for i in range(0, 100): + node1.execute('insert into test values ({0}, {0})'.format(i)) + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(i, i, )]) + node1.execute('delete from test') + + def test_logical_replication_fail(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") + + with __class__.helper__get_node(node_svc) as node: + with pytest.raises(expected_exception=InitNodeException): + node.init(allow_logical=True) + + def test_replication_slots(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.replicate(slot='slot1').start() as replica: + replica.execute('select 1') + + # cannot create new slot with the same name + with pytest.raises(expected_exception=testgres_TestgresException): + node.replicate(slot='slot1') + + def test_incorrect_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + # node has no master, can't catch up + with pytest.raises(expected_exception=testgres_TestgresException): + node.catchup() + + def test_promotion(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: + master.init().start() + master.safe_psql('create table abc(id serial)') + + with master.replicate().start() as replica: + master.stop() + replica.promote() + + # make standby becomes writable master + replica.safe_psql('insert into abc values (1)') + res = replica.safe_psql('select * from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_dump(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + query_create = 'create table test as select generate_series(1, 2) as val' + query_select = 'select * from test order by val asc' + + with __class__.helper__get_node(node_svc).init().start() as node1: + + node1.execute(query_create) + for format in ['plain', 'custom', 'directory', 'tar']: + with removing(node_svc.os_ops, node1.dump(format=format)) as dump: + with __class__.helper__get_node(node_svc).init().start() as node3: + if format == 'directory': + assert (os.path.isdir(dump)) + else: + assert (os.path.isfile(dump)) + # restore dump + node3.restore(filename=dump) + res = node3.execute(query_select) + assert (res == [(1, ), (2, )]) + + def test_pgbench(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pgbench") + + with __class__.helper__get_node(node_svc).init().start() as node: + # initialize pgbench DB and run benchmarks + node.pgbench_init( + scale=2, + foreign_keys=True, + options=['-q'] + ).pgbench_run(time=2) + + # run TPC-B benchmark + proc = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) + out = proc.communicate()[0] + assert (b'tps = ' in out) + + def test_unix_sockets(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init(unix_sockets=False, allow_streaming=True) + node.start() + + res_exec = node.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = node.safe_psql('select 1') + assert (res_psql == b'1\n') + + with node.replicate() as r: + assert type(r) == PostgresNode # noqa: E721 + r.start() + res_exec = r.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = r.safe_psql('select 1') + assert (res_psql == b'1\n') + + def test_the_same_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + assert (node._should_free_port) + assert (type(node.port) == int) # noqa: E721 + node_port_copy = node.port + r = node.safe_psql("SELECT 1;") + assert (__class__.helper__rm_carriage_returns(r) == b'1\n') + + with __class__.helper__get_node(node_svc, port=node.port) as node2: + assert (type(node2.port) == int) # noqa: E721 + assert (node2.port == node.port) + assert not (node2._should_free_port) + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): + node2.init().start() + + # node is still working + assert (node.port == node_port_copy) + assert (node._should_free_port) + r = node.safe_psql("SELECT 3;") + assert (__class__.helper__rm_carriage_returns(r) == b'3\n') + + class tagPortManagerProxy(PortManager): + m_PrevPortManager: PortManager + + m_DummyPortNumber: int + m_DummyPortMaxUsage: int + + m_DummyPortCurrentUsage: int + m_DummyPortTotalUsage: int + + def __init__(self, prevPortManager: PortManager, dummyPortNumber: int, dummyPortMaxUsage: int): + assert isinstance(prevPortManager, PortManager) + assert type(dummyPortNumber) == int # noqa: E721 + assert type(dummyPortMaxUsage) == int # noqa: E721 + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + super().__init__() + + self.m_PrevPortManager = prevPortManager + + self.m_DummyPortNumber = dummyPortNumber + self.m_DummyPortMaxUsage = dummyPortMaxUsage + + self.m_DummyPortCurrentUsage = 0 + self.m_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert self.m_DummyPortCurrentUsage == 0 + + assert self.m_PrevPortManager is not None + + def reserve_port(self) -> int: + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortTotalUsage == self.m_DummyPortMaxUsage: + return self.m_PrevPortManager.reserve_port() + + self.m_DummyPortTotalUsage += 1 + self.m_DummyPortCurrentUsage += 1 + return self.m_DummyPortNumber + + def release_port(self, dummyPortNumber: int): + assert type(dummyPortNumber) == int # noqa: E721 + + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortCurrentUsage > 0 and dummyPortNumber == self.m_DummyPortNumber: + assert self.m_DummyPortTotalUsage > 0 + self.m_DummyPortCurrentUsage -= 1 + return + + return self.m_PrevPortManager.release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert PostgresNode._C_MAX_START_ATEMPTS == 5 + + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + node2.init().start() + + assert node2.port != node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 0 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert node2.is_started + r = node2.safe_psql("SELECT 2;") + assert __class__.helper__rm_carriage_returns(r) == b'2\n' + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_port_conflict(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = PostgresNode._C_MAX_START_ATEMPTS + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts.") + ): + node2.init().start() + + assert node2.port == node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 1 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert not node2.is_started + + # node2 must release our dummyPort (node1.port) + assert (proxy.m_DummyPortCurrentUsage == 0) + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node1.port is not None + assert type(node1.port) == int # noqa: E721 + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int # noqa: E721 + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("try to get node2.port...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("PostgresNode port is not defined.") + "$" + ): + p = node2.port + assert p is None + + def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node1.port is not None + assert type(node1.port) == int # noqa: E721 + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int # noqa: E721 + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("node2 is trying to start...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("Can't start PostgresNode. Port is not defined.") + "$" + ): + node2.start() + + @staticmethod + def helper__get_node( + node_svc: PostgresNodeService, + name: typing.Optional[str] = None, + port: typing.Optional[int] = None, + port_manager: typing.Optional[PortManager] = None + ) -> PostgresNode: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node_svc.os_ops, OsOperations) + assert isinstance(node_svc.port_manager, PortManager) + + if port_manager is None: + port_manager = node_svc.port_manager + + return PostgresNode( + name, + port=port, + os_ops=node_svc.os_ops, + port_manager=port_manager if port is None else None + ) + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + if not __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires {0}+'.format(ver2)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(ver1: str, ver2: str): + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + if __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires <{0}'.format(ver2)) + + @staticmethod + def helper__pg_version_ge(ver1: str, ver2: str) -> bool: + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + v1 = PgVer(ver1) + v2 = PgVer(ver2) + return v1 >= v2 + + @staticmethod + def helper__rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if isinstance(out, (int, float, complex)): + return out + + if isinstance(out, tuple): + return tuple(__class__.helper__rm_carriage_returns(item) for item in out) + + if isinstance(out, bytes): + return out.replace(b'\r', b'') + + assert type(out) == str # noqa: E721 + return out.replace('\r', '') + + @staticmethod + def helper__skip_test_if_util_not_exist(os_ops: OsOperations, name: str): + assert isinstance(os_ops, OsOperations) + assert type(name) == str # noqa: E721 + if not __class__.helper__util_exists(os_ops, name): + pytest.skip('might be missing') + + @staticmethod + def helper__util_exists(os_ops: OsOperations, util): + assert isinstance(os_ops, OsOperations) + + def good_properties(f): + return (os_ops.path_exists(f) and # noqa: W504 + os_ops.isfile(f) and # noqa: W504 + os_ops.is_executable(f)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path2(os_ops, util)): + return True + + # check if util is in PATH + for path in os_ops.environ("PATH").split(os.pathsep): + if good_properties(os.path.join(path, util)): + return True diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py new file mode 100644 index 00000000..63e5f37e --- /dev/null +++ b/tests/test_testgres_local.py @@ -0,0 +1,414 @@ +# coding: utf-8 +import os +import re +import subprocess +import pytest +import psutil +import platform +import logging + +import testgres + +from testgres import StartNodeException +from testgres import ExecUtilException +from testgres import NodeApp +from testgres import scoped_config +from testgres import get_new_node +from testgres import get_bin_path +from testgres import get_pg_config +from testgres import get_pg_version + +# NOTE: those are ugly imports +from testgres.utils import bound_ports +from testgres.utils import PgVer +from testgres.node import ProcessProxy + + +def pg_version_ge(version): + cur_ver = PgVer(get_pg_version()) + min_ver = PgVer(version) + return cur_ver >= min_ver + + +def util_exists(util): + def good_properties(f): + return (os.path.exists(f) and # noqa: W504 + os.path.isfile(f) and # noqa: W504 + os.access(f, os.X_OK)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path(util)): + return True + + # check if util is in PATH + for path in os.environ["PATH"].split(os.pathsep): + if good_properties(os.path.join(path, util)): + return True + + +def rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if os.name == 'nt': + if isinstance(out, (int, float, complex)): + return out + elif isinstance(out, tuple): + return tuple(rm_carriage_returns(item) for item in out) + elif isinstance(out, bytes): + return out.replace(b'\r', b'') + else: + return out.replace('\r', '') + else: + return out + + +class TestTestgresLocal: + def test_pg_config(self): + # check same instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config() + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config() + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) != id(b)) + + def test_ports_management(self): + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + if len(bound_ports) != 0: + logging.warning("bound_ports is not empty: {0}".format(bound_ports)) + + stage0__bound_ports = bound_ports.copy() + + with get_new_node() as node: + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + assert node.port is not None + assert type(node.port) == int # noqa: E721 + + logging.info("node port is {0}".format(node.port)) + + assert node.port in bound_ports + assert node.port not in stage0__bound_ports + + assert stage0__bound_ports <= bound_ports + assert len(stage0__bound_ports) + 1 == len(bound_ports) + + stage1__bound_ports = stage0__bound_ports.copy() + stage1__bound_ports.add(node.port) + + assert stage1__bound_ports == bound_ports + + # check that port has been freed successfully + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + assert bound_ports == stage0__bound_ports + + def test_child_process_dies(self): + # test for FileNotFound exception during child_processes() function + cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] + + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break + + def test_upgrade_node(self): + old_bin_dir = os.path.dirname(get_bin_path("pg_config")) + new_bin_dir = os.path.dirname(get_bin_path("pg_config")) + with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old: + node_old.init() + node_old.start() + node_old.stop() + with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new: + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + assert (b'Upgrade Complete' in res) + + class tagPortManagerProxy: + sm_prev_testgres_reserve_port = None + sm_prev_testgres_release_port = None + + sm_DummyPortNumber = None + sm_DummyPortMaxUsage = None + + sm_DummyPortCurrentUsage = None + sm_DummyPortTotalUsage = None + + def __init__(self, dummyPortNumber, dummyPortMaxUsage): + assert type(dummyPortNumber) == int # noqa: E721 + assert type(dummyPortMaxUsage) == int # noqa: E721 + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + assert __class__.sm_prev_testgres_reserve_port is None + assert __class__.sm_prev_testgres_release_port is None + assert testgres.utils.reserve_port == testgres.utils.internal__reserve_port + assert testgres.utils.release_port == testgres.utils.internal__release_port + + __class__.sm_prev_testgres_reserve_port = testgres.utils.reserve_port + __class__.sm_prev_testgres_release_port = testgres.utils.release_port + + testgres.utils.reserve_port = __class__._proxy__reserve_port + testgres.utils.release_port = __class__._proxy__release_port + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + __class__.sm_DummyPortNumber = dummyPortNumber + __class__.sm_DummyPortMaxUsage = dummyPortMaxUsage + + __class__.sm_DummyPortCurrentUsage = 0 + __class__.sm_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert __class__.sm_DummyPortCurrentUsage == 0 + + assert __class__.sm_prev_testgres_reserve_port is not None + assert __class__.sm_prev_testgres_release_port is not None + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + testgres.utils.reserve_port = __class__.sm_prev_testgres_reserve_port + testgres.utils.release_port = __class__.sm_prev_testgres_release_port + + __class__.sm_prev_testgres_reserve_port = None + __class__.sm_prev_testgres_release_port = None + + @staticmethod + def _proxy__reserve_port(): + assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortCurrentUsage) == int # noqa: E721 + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_reserve_port is not None + + if __class__.sm_DummyPortTotalUsage == __class__.sm_DummyPortMaxUsage: + return __class__.sm_prev_testgres_reserve_port() + + __class__.sm_DummyPortTotalUsage += 1 + __class__.sm_DummyPortCurrentUsage += 1 + return __class__.sm_DummyPortNumber + + @staticmethod + def _proxy__release_port(dummyPortNumber): + assert type(dummyPortNumber) == int # noqa: E721 + + assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortCurrentUsage) == int # noqa: E721 + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_release_port is not None + + if __class__.sm_DummyPortCurrentUsage > 0 and dummyPortNumber == __class__.sm_DummyPortNumber: + assert __class__.sm_DummyPortTotalUsage > 0 + __class__.sm_DummyPortCurrentUsage -= 1 + return + + return __class__.sm_prev_testgres_release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5 + + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with get_new_node() as node1: + node1.init().start() + assert (node1._should_free_port) + assert (type(node1.port) == int) # noqa: E721 + node1_port_copy = node1.port + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + assert (node2._should_free_port) + assert (node2.port == node1.port) + + node2.init().start() + + assert (node2.port != node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert (node2.is_started) + + assert (rm_carriage_returns(node2.safe_psql("SELECT 2;")) == b'2\n') + + # node1 is still working + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') + + def test_port_conflict(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS + + with get_new_node() as node1: + node1.init().start() + assert (node1._should_free_port) + assert (type(node1.port) == int) # noqa: E721 + node1_port_copy = node1.port + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + assert (node2._should_free_port) + assert (node2.port == node1.port) + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts.") + ): + node2.init().start() + + assert (node2.port == node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert not (node2.is_started) + + # node2 must release our dummyPort (node1.port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + + # node1 is still working + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') + + def test_simple_with_bin_dir(self): + with get_new_node() as node: + node.init().start() + bin_dir = node.bin_dir + + app = NodeApp() + with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir: + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() + + while True: + try: + app.make_simple(base_dir=node.base_dir, bin_dir="wrong/path") + except FileNotFoundError: + break # Expected error + except ExecUtilException: + break # Expected error + + raise RuntimeError("Error was expected.") # We should not reach this + + return + + def test_set_auto_conf(self): + # elements contain [property id, value, storage value] + testData = [ + ["archive_command", + "cp '%p' \"/mnt/server/archivedir/%f\"", + "'cp \\'%p\\' \"/mnt/server/archivedir/%f\""], + ["log_line_prefix", + "'\n\r\t\b\\\"", + "'\\\'\\n\\r\\t\\b\\\\\""], + ["log_connections", + True, + "on"], + ["log_disconnections", + False, + "off"], + ["autovacuum_max_workers", + 3, + "3"] + ] + if pg_version_ge('12'): + testData.append(["restore_command", + 'cp "/mnt/server/archivedir/%f" \'%p\'', + "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"]) + + with get_new_node() as node: + node.init().start() + + options = {} + + for x in testData: + options[x[0]] = x[1] + + node.set_auto_conf(options) + node.stop() + node.slow_start() + + auto_conf_path = f"{node.data_dir}/postgresql.auto.conf" + with open(auto_conf_path, "r") as f: + content = f.read() + + for x in testData: + assert x[0] + " = " + x[2] in content + + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + + if platform.system().lower() == "windows": + name2 = name + ".exe" + else: + name2 = name + + if not util_exists(name2): + pytest.skip('might be missing') diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py new file mode 100755 index 00000000..6a8d068b --- /dev/null +++ b/tests/test_testgres_remote.py @@ -0,0 +1,190 @@ +# coding: utf-8 +import os + +import pytest +import logging + +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices + +import testgres + +from testgres.exceptions import InitNodeException +from testgres.exceptions import ExecUtilException + +from testgres.config import scoped_config +from testgres.config import testgres_config + +from testgres import get_bin_path +from testgres import get_pg_config + +# NOTE: those are ugly imports + + +def util_exists(util): + def good_properties(f): + return (testgres_config.os_ops.path_exists(f) and # noqa: W504 + testgres_config.os_ops.isfile(f) and # noqa: W504 + testgres_config.os_ops.is_executable(f)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path(util)): + return True + + # check if util is in PATH + for path in testgres_config.os_ops.environ("PATH").split(testgres_config.os_ops.pathsep): + if good_properties(os.path.join(path, util)): + return True + + +class TestTestgresRemote: + @pytest.fixture(autouse=True, scope="class") + def implicit_fixture(self): + cur_os_ops = PostgresNodeServices.sm_remote.os_ops + assert cur_os_ops is not None + + prev_ops = testgres_config.os_ops + assert prev_ops is not None + testgres_config.set_os_ops(os_ops=cur_os_ops) + assert testgres_config.os_ops is cur_os_ops + yield + assert testgres_config.os_ops is cur_os_ops + testgres_config.set_os_ops(os_ops=prev_ops) + assert testgres_config.os_ops is prev_ops + + def test_init__LANG_ะก(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + + try: + os.environ["LANG"] = "C" + + with __class__.helper__get_node() as node: + node.init().start() + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + + def test_init__unk_LANG_and_LC_CTYPE(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + prev_LANGUAGE = os.environ.get("LANGUAGE") + prev_LC_CTYPE = os.environ.get("LC_CTYPE") + prev_LC_COLLATE = os.environ.get("LC_COLLATE") + + try: + # TODO: Pass unkData through test parameter. + unkDatas = [ + ("UNKNOWN_LANG", "UNKNOWN_CTYPE"), + ("\"UNKNOWN_LANG\"", "\"UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG\\", "\\UNKNOWN_CTYPE\\"), + ("\"UNKNOWN_LANG", "UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG", "UNKNOWN_CTYPE\\"), + ("\\", "\\"), + ("\"", "\""), + ] + + errorIsDetected = False + + for unkData in unkDatas: + logging.info("----------------------") + logging.info("Unk LANG is [{0}]".format(unkData[0])) + logging.info("Unk LC_CTYPE is [{0}]".format(unkData[1])) + + os.environ["LANG"] = unkData[0] + os.environ.pop("LANGUAGE", None) + os.environ["LC_CTYPE"] = unkData[1] + os.environ.pop("LC_COLLATE", None) + + assert os.environ.get("LANG") == unkData[0] + assert not ("LANGUAGE" in os.environ.keys()) + assert os.environ.get("LC_CTYPE") == unkData[1] + assert not ("LC_COLLATE" in os.environ.keys()) + + assert os.getenv('LANG') == unkData[0] + assert os.getenv('LANGUAGE') is None + assert os.getenv('LC_CTYPE') == unkData[1] + assert os.getenv('LC_COLLATE') is None + + exc: ExecUtilException = None + with __class__.helper__get_node() as node: + try: + node.init() # IT RAISES! + except InitNodeException as e: + exc = e.__cause__ + assert exc is not None + assert isinstance(exc, ExecUtilException) + + if exc is None: + logging.warning("We expected an error!") + continue + + errorIsDetected = True + + assert isinstance(exc, ExecUtilException) + + errMsg = str(exc) + logging.info("Error message is {0}: {1}".format(type(exc).__name__, errMsg)) + + assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + ")" in errMsg + assert "initdb: error: invalid locale settings; check LANG and LC_* environment variables" in errMsg + continue + + if not errorIsDetected: + pytest.xfail("All the bad data are processed without errors!") + + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) + __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) + __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) + + def test_pg_config(self): + # check same instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config() + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config() + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) != id(b)) + + @staticmethod + def helper__get_node(name=None): + svc = PostgresNodeServices.sm_remote + + assert isinstance(svc, PostgresNodeService) + assert isinstance(svc.os_ops, testgres.OsOperations) + assert isinstance(svc.port_manager, testgres.PortManager) + + return testgres.PostgresNode( + name, + os_ops=svc.os_ops, + port_manager=svc.port_manager) + + @staticmethod + def helper__restore_envvar(name, prev_value): + if prev_value is None: + os.environ.pop(name, None) + else: + os.environ[name] = prev_value + + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + if not util_exists(name): + pytest.skip('might be missing') diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..39e9dda0 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,63 @@ +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations + +from testgres.utils import parse_pg_version +from testgres.utils import get_pg_config2 +from testgres import scoped_config + +import pytest +import typing + + +class TestUtils: + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_parse_pg_version(self): + # Linux Mint + assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" + # Linux Ubuntu + assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" + # Windows + assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" + # Macos + assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + + def test_get_pg_config2(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # check same instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config2(os_ops, None) + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config2(os_ops, None) + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) != id(b))