ci: Make compute resources for CI configurable

See prior commit for an explanation for the goal of the change and why it had
to be split into two commits.

Reviewed-by: Daniel Gustafsson <[email protected]>
Reviewed-by: Nazir Bilal Yavuz <[email protected]>
Discussion: https://postgr.es/m/20230808021541[email protected]
Back: 15-, where CI support was added

diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index f4276ad8692c95d916dcb98b16e303b64093187b..0cf7ba779964e206c99a1143492e18b3734e3a27 100644 (file)
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -5,12 +5,6 @@
 
 
 env:
-  # Source of images / containers
-  GCP_PROJECT: pg-ci-images
-  IMAGE_PROJECT: $GCP_PROJECT
-  CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci
-  DISK_SIZE: 25
-
   # The lower depth accelerates git clone. Use a bit of depth so that
   # concurrent tasks and retrying older jobs have a chance of working.
   CIRRUS_CLONE_DEPTH: 500
@@ -29,45 +23,6 @@ env:
   PG_TEST_EXTRA: kerberos ldap ssl load_balance
 
 
-# Define how to run various types of tasks.
-
-# VMs provided by cirrus-ci. Each user has a limited number of "free" credits
-# for testing.
-cirrus_community_vm_template: &cirrus_community_vm_template
-  compute_engine_instance:
-    image_project: $IMAGE_PROJECT
-    image: family/$IMAGE_FAMILY
-    platform: $PLATFORM
-    cpu: $CPUS
-    disk: $DISK_SIZE
-
-
-default_linux_task_template: &linux_task_template
-  env:
-    PLATFORM: linux
-  <<: *cirrus_community_vm_template
-
-
-default_freebsd_task_template: &freebsd_task_template
-  env:
-    PLATFORM: freebsd
-  <<: *cirrus_community_vm_template
-
-
-default_windows_task_template: &windows_task_template
-  env:
-    PLATFORM: windows
-  <<: *cirrus_community_vm_template
-
-
-# macos workers provided by cirrus-ci
-default_macos_task_template: &macos_task_template
-  env:
-    PLATFORM: macos
-  macos_instance:
-    image: $IMAGE
-
-
 # What files to preserve in case tests fail
 on_failure_ac: &on_failure_ac
   log_artifacts:

diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644 (file)index 0000000..a83129a
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,73 @@
+# CI configuration file for CI utilizing cirrus-ci.org
+#
+# For instructions on how to enable the CI integration in a repository and
+# further details, see src/tools/ci/README
+#
+#
+# The actual CI tasks are defined in .cirrus.tasks.yml. To make the compute
+# resources for CI configurable on a repository level, the "final" CI
+# configuration is the combination of:
+#
+# 1) the contents of this file
+#
+# 2) if defined, the contents of the file referenced by the, repository
+#    level, REPO_CI_CONFIG_GIT_URL variable (see
+#    https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted
+#    format)
+#
+# 3) .cirrus.tasks.yml
+#
+# This composition is done by .cirrus.star
+
+
+env:
+  # Source of images / containers
+  GCP_PROJECT: pg-ci-images
+  IMAGE_PROJECT: $GCP_PROJECT
+  CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci
+  DISK_SIZE: 25
+
+
+# Define how to run various types of tasks.
+
+# VMs provided by cirrus-ci. Each user has a limited number of "free" credits
+# for testing.
+cirrus_community_vm_template: &cirrus_community_vm_template
+  compute_engine_instance:
+    image_project: $IMAGE_PROJECT
+    image: family/$IMAGE_FAMILY
+    platform: $PLATFORM
+    cpu: $CPUS
+    disk: $DISK_SIZE
+
+
+default_linux_task_template: &linux_task_template
+  env:
+    PLATFORM: linux
+  <<: *cirrus_community_vm_template
+
+
+default_freebsd_task_template: &freebsd_task_template
+  env:
+    PLATFORM: freebsd
+  <<: *cirrus_community_vm_template
+
+
+default_windows_task_template: &windows_task_template
+  env:
+    PLATFORM: windows
+  <<: *cirrus_community_vm_template
+
+
+# macos workers provided by cirrus-ci
+default_macos_task_template: &macos_task_template
+  env:
+    PLATFORM: macos
+  macos_instance:
+    image: $IMAGE
+
+
+# Contents of REPO_CI_CONFIG_GIT_URL, if defined, will be inserted here,
+# followed by the contents .cirrus.tasks.yml. This allows
+# REPO_CI_CONFIG_GIT_URL to override how the task types above will be
+# executed, e.g. using a custom compute account or permanent workers.

diff --git a/src/tools/ci/README b/src/tools/ci/README
index 80d01939e847f3632556ca3f87d0ae5579745339..30ddd200c9669bc05729847fc12cb2e721bfd956 100644 (file)
--- a/src/tools/ci/README
+++ b/src/tools/ci/README
@@ -65,3 +65,20 @@ messages. Currently the following controls are available:
 
   Only runs CI on operating systems specified. This can be useful when
   addressing portability issues affecting only a subset of platforms.
+
+
+Using custom compute resources for CI
+=====================================
+
+When running a lot of tests in a repository, cirrus-ci's free credits do not
+suffice. In those cases a repository can be configured to use other
+infrastructure for running tests. To do so, the REPO_CI_CONFIG_GIT_URL
+variable can be configured for the repository in the cirrus-ci web interface,
+at https://cirrus-ci.com//<user or organization>. The file referenced
+(see https://cirrus-ci.org/guide/programming-tasks/#fs) by the variable can
+overwrite the default execution method for different operating systems,
+defined in .cirrus.yml, by redefining the relevant yaml anchors.
+
+Custom compute resources can be provided using
+- https://cirrus-ci.org/guide/supported-computing-services/
+- https://cirrus-ci.org/guide/persistent-workers/