fix postgres cluster

Author: JuArce

infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini

@@ -1,4 +1,4 @@
-[DEFAULT]
+[global]
 # ============================================
 # Hoodi Environment Configuration
 # ============================================

infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini

@@ -1,4 +1,4 @@
-[DEFAULT]
+[global]
 # ============================================
 # Mainnet Environment Configuration
 # ============================================

infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml

@@ -9,6 +9,7 @@
           - postgresql-common
           - ca-certificates
           - curl
+          - acl
         state: latest
         update_cache: true
       vars:

infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml

@@ -1,19 +1,33 @@
+- name: Import pg_autofailover_common playbook
+  ansible.builtin.import_playbook: pg_autofailover_common.yaml
+  vars:
+    host: "{{ host }}"
+
 - name: PostgreSQL Monitor Setup
   hosts: "{{ host }}"
+  vars:
+    ansible_common_remote_group: postgres
 
   tasks:
-    - name: Import pg_autofailover_common playbook
-      ansible.builtin.import_playbook: pg_autofailover_common.yaml
-      vars:
-        host: "{{ host }}"
+    - name: Set config file path
+      set_fact:
+        config_file: "ini/config-{{ env }}.ini"
 
-- hosts: "{{ host }}"
-  vars:
-    db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}"
-    monitor_pgdata: "{{ lookup('ini', 'monitor_pgdata', file='ini/config-{{ env }}.ini', default='/var/lib/postgresql/monitor') }}"
-    monitor_port: "{{ lookup('ini', 'monitor_port', file='ini/config-{{ env }}.ini', default='5432') }}"
+    - name: Debug config file
+      debug:
+        msg: "Using config file: {{ config_file }}"
+
+    - name: Set config vars from INI file
+      set_fact:
+        db_password: "{{ lookup('ini', 'db_password', file=config_file) }}"
+        monitor_pgdata: "{{ lookup('ini', 'monitor_pgdata', file=config_file, default='/var/lib/postgresql/monitor') }}"
+        monitor_port: "{{ lookup('ini', 'monitor_port', file=config_file, default='5432') }}"
+      no_log: true
+
+    - name: Debug vars
+      debug:
+        msg: "env={{ env }}, monitor_pgdata={{ monitor_pgdata }}"
 
-  tasks:
     - name: Install postgresql-16-auto-failover
       become: true
       apt:
@@ -50,19 +64,6 @@
       args:
         creates: "{{ monitor_pgdata }}/PG_VERSION"
 
-    - name: Wait for monitor to be ready
-      wait_for:
-        port: "{{ monitor_port }}"
-        delay: 5
-        timeout: 60
-
-    - name: Set password for autoctl_node user
-      become: true
-      become_user: postgres
-      shell: |
-        psql -d pg_auto_failover -c "ALTER USER autoctl_node PASSWORD '{{ db_password }}';"
-      no_log: true
-
     - name: Update pg_hba.conf for Tailscale network
       become: true
       become_user: postgres
@@ -99,3 +100,16 @@
         daemon_reload: true
       vars:
         ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Wait for monitor to be ready
+      wait_for:
+        port: "{{ monitor_port }}"
+        delay: 5
+        timeout: 60
+
+    - name: Set password for autoctl_node user
+      become: true
+      become_user: postgres
+      shell: |
+        psql -d pg_auto_failover -c "ALTER USER autoctl_node PASSWORD '{{ db_password }}';"
+      no_log: true

infra/aggregation_mode/ansible/playbooks/pg_node.yaml

@@ -1,23 +1,29 @@
+- name: Import pg_autofailover_common playbook
+  ansible.builtin.import_playbook: pg_autofailover_common.yaml
+  vars:
+    host: "{{ host }}"
+
 - name: PostgreSQL Node Setup
   hosts: "{{ host }}"
+  vars:
+    ansible_common_remote_group: postgres
 
   tasks:
-    - name: Import pg_autofailover_common playbook
-      ansible.builtin.import_playbook: pg_autofailover_common.yaml
-      vars:
-        host: "{{ host }}"
+    - name: Set config file path
+      set_fact:
+        config_file: "ini/config-{{ env }}.ini"
 
-- hosts: "{{ host }}"
-  vars:
-    db_name: "{{ lookup('ini', 'db_name', file='ini/config-{{ env }}.ini') }}"
-    db_user: "{{ lookup('ini', 'db_user', file='ini/config-{{ env }}.ini') }}"
-    db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}"
-    postgres_monitor_hostname: "{{ lookup('ini', 'postgres_monitor_hostname', file='ini/config-{{ env }}.ini') }}"
-    node_pgdata: "{{ lookup('ini', 'node_pgdata', file='ini/config-{{ env }}.ini', default='/var/lib/postgresql/node') }}"
-    node_port: "{{ lookup('ini', 'node_port', file='ini/config-{{ env }}.ini', default='5432') }}"
-    backup_dir: "{{ lookup('ini', 'backup_dir', file='ini/config-{{ env }}.ini', default='/var/lib/backup') }}"
+    - name: Set config vars from INI file
+      set_fact:
+        db_name: "{{ lookup('ini', 'db_name', file=config_file) }}"
+        db_user: "{{ lookup('ini', 'db_user', file=config_file) }}"
+        db_password: "{{ lookup('ini', 'db_password', file=config_file) }}"
+        postgres_monitor_hostname: "{{ lookup('ini', 'postgres_monitor_hostname', file=config_file) }}"
+        node_pgdata: "{{ lookup('ini', 'node_pgdata', file=config_file, default='/var/lib/postgresql/node') }}"
+        node_port: "{{ lookup('ini', 'node_port', file=config_file, default='5432') }}"
+        backup_dir: "{{ lookup('ini', 'backup_dir', file=config_file, default='/var/lib/backup') }}"
+      no_log: true
 
-  tasks:
     - name: Create backup directory
       become: true
       file:
@@ -36,6 +42,10 @@
       become: true
       become_user: postgres
 
+    - name: Debug node_initialized
+      debug:
+        msg: "Node initialized: {{ node_initialized.stat.exists }}, path: {{ node_pgdata }}/PG_VERSION"
+
     - name: Get Tailscale IP
       shell: tailscale ip --4
       register: tailscale_ip
@@ -56,7 +66,6 @@
           --hostname {{ tailscale_ip.stdout }} \
           --monitor 'postgres://{{ db_user }}:{{ db_password }}@{{ postgres_monitor_hostname }}:5432/pg_auto_failover?sslmode=require'
       when: not node_initialized.stat.exists
-      no_log: true
       args:
         creates: "{{ node_pgdata }}/PG_VERSION"
 
@@ -108,3 +117,19 @@
         port: "{{ node_port }}"
         delay: 5
         timeout: 60
+
+    - name: Check if node is writable (primary)
+      become: true
+      become_user: postgres
+      shell: |
+        psql -d {{ db_name }} -tAc "SELECT NOT pg_is_in_recovery();"
+      register: is_writable
+      changed_when: false
+
+    - name: Set password for pgautofailover_replicator user
+      become: true
+      become_user: postgres
+      shell: |
+        psql -d {{ db_name }} -c "ALTER USER pgautofailover_replicator PASSWORD '{{ db_password }}';"
+      when: is_writable.stdout == 't'
+      no_log: true

infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml

@@ -1,24 +1,23 @@
-- name: Deploy PostgreSQL Auto-Failover Cluster
-  hosts: localhost
-  gather_facts: no
+- name: Deploy PostgreSQL Monitor
+  ansible.builtin.import_playbook: pg_monitor.yaml
+  vars:
+    host: postgres_monitor
+    env: "{{ env }}"
 
-  tasks:
-    - name: Deploy PostgreSQL Monitor
-      ansible.builtin.import_playbook: pg_monitor.yaml
-      vars:
-        host: postgres_monitor
+- name: Deploy PostgreSQL Primary Node
+  ansible.builtin.import_playbook: pg_node.yaml
+  vars:
+    host: postgres_primary
+    env: "{{ env }}"
 
-    - name: Deploy PostgreSQL Primary Node
-      ansible.builtin.import_playbook: pg_node.yaml
-      vars:
-        host: postgres_primary
+- name: Deploy PostgreSQL Secondary Node
+  ansible.builtin.import_playbook: pg_node.yaml
+  vars:
+    host: postgres_secondary
+    env: "{{ env }}"
 
-    - name: Deploy PostgreSQL Secondary Node
-      ansible.builtin.import_playbook: pg_node.yaml
-      vars:
-        host: postgres_secondary
-
-    - name: Run Database Migrations
-      ansible.builtin.import_playbook: postgres_migrations.yaml
-      vars:
-        host: postgres_primary
+- name: Run Database Migrations
+  ansible.builtin.import_playbook: postgres_migrations.yaml
+  vars:
+    host: postgres_primary
+    env: "{{ env }}"

infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml

@@ -1,26 +1,32 @@
+- name: Import setup playbook
+  ansible.builtin.import_playbook: setup.yaml
+  vars:
+    host: "{{ host }}"
+    env: "{{ env }}"
+
+- name: Import rust playbook
+  ansible.builtin.import_playbook: rust.yaml
+  vars:
+    host: "{{ host }}"
+    env: "{{ env }}"
+
 - name: PostgreSQL Migrations Setup
   hosts: "{{ host }}"
 
   tasks:
-    - name: Import setup playbook
-      ansible.builtin.import_playbook: setup.yaml
-      vars:
-        host: "{{ host }}"
+    - name: Set config file path
+      set_fact:
+        config_file: "ini/config-{{ env }}.ini"
 
-    - name: Import rust playbook
-      ansible.builtin.import_playbook: rust.yaml
-      vars:
-        host: "{{ host }}"
-
-- hosts: "{{ host }}"
-  vars:
-    db_name: "{{ lookup('ini', 'db_name', file='ini/config-{{ env }}.ini') }}"
-    db_user: "{{ lookup('ini', 'db_user', file='ini/config-{{ env }}.ini') }}"
-    db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}"
-    postgres_primary_hostname: "{{ lookup('ini', 'postgres_primary_hostname', file='ini/config-{{ env }}.ini') }}"
-    git_branch: "{{ lookup('ini', 'git_branch', file='ini/config-{{ env }}.ini') }}"
+    - name: Set config vars from INI file
+      set_fact:
+        db_name: "{{ lookup('ini', 'db_name', file=config_file) }}"
+        db_user: "{{ lookup('ini', 'db_user', file=config_file) }}"
+        db_password: "{{ lookup('ini', 'db_password', file=config_file) }}"
+        postgres_primary_hostname: "{{ lookup('ini', 'postgres_primary_hostname', file=config_file) }}"
+        git_branch: "{{ lookup('ini', 'git_branch', file=config_file) }}"
+      no_log: true
 
-  tasks:
     - name: Clone aligned_layer repository
       git:
         repo: https://github.com/yetanotherco/aligned_layer.git

infra/aggregation_mode/ansible/playbooks/rust.yaml

@@ -1,7 +1,7 @@
 - name: Rust Setup
   hosts: "{{ host }}"
   vars:
-    rust_version: 1.83.0
+    rust_version: 1.92.0
 
   tasks:
     - name: Check if cargo is installed