Replace manual escluster setup with script

Author: cbaxley

.github/workflows/escluster.yml

@@ -87,45 +87,43 @@ jobs:
           cd testing/v2/development
           docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline
 
-      - name: Install Python requirements
-        run: |
-          cd testing/v2/development
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd /home/lme-user/LME/testing/v2/installers/azure && \
-            pip install -r requirements.txt
-          "
-
-      - name: Build Azure Ubuntu instances with Windows VM
+      - name: Set up cluster via setup_cluster.sh
         env:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
         run: |
           cd testing/v2/development
+          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
+
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
+            printf '%s\n' \
+              'export RESOURCE_GROUP=pipe-${{ env.UNIQUE_ID }}' \
+              'export PUBLIC_IP=${{ env.IP_ADDRESS }}/32' \
+              'export VM_SIZE=Standard_D8_v4' \
+              'export LOCATION=${{ inputs.azure_region || '\''centralus'\'' }}' \
+              'export AUTO_SHUTDOWN_TIME=23:00' \
+              'export LME_USER=lme-user' \
+              'export BRANCH=${{ env.BRANCH_NAME }}' \
+              'export CLUSTER_SIZE=3' \
+              > ${INSTALLERS_DIR}/exporter.txt && \
+            echo '=== exporter.txt ===' && cat ${INSTALLERS_DIR}/exporter.txt
+          "
+
           docker compose -p ${{ env.UNIQUE_ID }} exec -T \
             -e AZURE_CLIENT_ID \
             -e AZURE_CLIENT_SECRET \
             -e AZURE_TENANT_ID \
             -e AZURE_SUBSCRIPTION_ID \
             pipeline bash -c "
-              cd /home/lme-user/LME/testing/v2/installers && \
-              python3 ./azure/build_azure_linux_network.py \
-                -g pipe-${{ env.UNIQUE_ID }} \
-                -s ${{ env.IP_ADDRESS }}/32 \
-                -vs Standard_D8_v4 \
-                -l ${{ inputs.azure_region || 'centralus' }} \
-                -ast 23:00 \
-                -c 3 \
-                -w \
-                -y
+              cd /home/lme-user/LME/testing/v2/installers/cluster_installer && \
+              ./setup_cluster.sh
             "
 
       - name: Extract cluster metadata
         run: |
           cd testing/v2/development
-          echo "Waiting for VMs to boot..."
-          sleep 120
           INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
           MACHINES_FILE="${INSTALLERS_DIR}/pipe-${{ env.UNIQUE_ID }}.machines.json"
 
@@ -141,14 +139,10 @@ jobs:
             "jq -c '[.linux_vms[].private_ip]' ${MACHINES_FILE}" | tr -d '\r\n ')
           echo "ALL_PRIVATE_IPS_JSON=$ALL_PRIVATE_IPS_JSON" >> $GITHUB_ENV
 
-          WINDOWS_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            az network public-ip show --resource-group pipe-${{ env.UNIQUE_ID }} --name ws1-public-ip --query ipAddress --output tsv 2>/dev/null || \
-            az vm show --resource-group pipe-${{ env.UNIQUE_ID }} --name ws1 --show-details --query publicIps --output tsv 2>/dev/null || \
-            az vm list-ip-addresses --resource-group pipe-${{ env.UNIQUE_ID }} --query '[?virtualMachine.name==\`ws1\`].virtualMachine.network.publicIpAddresses[0].ipAddress' --output tsv 2>/dev/null || \
-            echo ''
-          " | tr -d '\r\n ')
+          WINDOWS_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c \
+            "cat ${INSTALLERS_DIR}/pipe-${{ env.UNIQUE_ID }}.ws1.ip.txt 2>/dev/null || echo ''" | tr -d '\r\n ')
           if [ -z "$WINDOWS_IP" ]; then
-            echo "Warning: Could not retrieve Windows VM public IP, using private IP instead"
+            echo "Warning: Could not read ws1 IP file, using private IP fallback"
             WINDOWS_IP="10.1.0.4"
           fi
           echo "WINDOWS_IP=$WINDOWS_IP" >> $GITHUB_ENV
@@ -158,268 +152,6 @@ jobs:
           echo "All private IPs: $ALL_PRIVATE_IPS_JSON"
           echo "Windows IP: $WINDOWS_IP"
 
-      - name: Set up SSH access to all Linux VMs
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd ${INSTALLERS_DIR} && \
-            PASSWORD=\$(cat pipe-${{ env.UNIQUE_ID }}.password.txt) && \
-            for IP in \$(jq -r '.linux_vms[].ip_address' pipe-${{ env.UNIQUE_ID }}.machines.json); do
-              echo \"Waiting for SSH on \$IP...\"
-              for i in \$(seq 1 30); do
-                if sshpass -p \"\$PASSWORD\" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 lme-user@\$IP 'echo ok' 2>/dev/null; then
-                  echo \"SSH ready on \$IP\"
-                  break
-                fi
-                echo -n '.'
-                sleep 10
-              done
-              echo \"Copying SSH key to \$IP...\"
-              ./lib/copy_ssh_key.sh lme-user \$IP pipe-${{ env.UNIQUE_ID }}.password.txt
-            done
-          "
-
-      - name: Set up SSH trust from master to child nodes
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-              'ssh-keygen -t rsa -b 4096 -N \"\" -f ~/.ssh/id_rsa -q <<< y 2>/dev/null || true'
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-              'sudo apt-get update -qq && sudo apt-get install -y -qq sshpass'
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd ${INSTALLERS_DIR} && \
-            PASSWORD=\$(cat pipe-${{ env.UNIQUE_ID }}.password.txt) && \
-            for ip in \$(jq -r '.linux_vms[1:][].private_ip' pipe-${{ env.UNIQUE_ID }}.machines.json); do
-              echo \"Copying master SSH key to \$ip...\"
-              ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-                \"sshpass -p '\$PASSWORD' ssh-copy-id -o StrictHostKeyChecking=no lme-user@\$ip\"
-              echo \"Testing SSH from master to \$ip...\"
-              ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-                \"ssh -o StrictHostKeyChecking=no lme-user@\$ip hostname\"
-            done
-          "
-
-      - name: Clone repo and prepare master for cluster install
-        run: |
-          cd testing/v2/development
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} '
-              git clone https://github.com/cisagov/LME.git ~/LME &&
-              cd ~/LME &&
-              git fetch --all --tags &&
-              if git show-ref --tags --verify --quiet \"refs/tags/${{ env.BRANCH_NAME }}\"; then
-                git checkout ${{ env.BRANCH_NAME }}
-              else
-                git checkout -t origin/${{ env.BRANCH_NAME }} || git checkout ${{ env.BRANCH_NAME }}
-              fi
-            '
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} '
-              cp ~/LME/config/example.env ~/LME/config/lme-environment.env &&
-              sed -i \"s/IPVAR=.*/IPVAR=${{ env.MASTER_PRIVATE_IP }}/\" ~/LME/config/lme-environment.env
-            '
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} '
-              sudo apt-get update && sudo apt-get install -y ansible jq
-            '
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} '
-              cd ~/LME/ansible && ansible-galaxy collection install -r requirements.yml --timeout 30
-            '
-          " || echo "Galaxy install failed, continuing with existing collections..."
-
-      - name: Run site.yml in cluster mode on master
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd ${INSTALLERS_DIR} && \
-            MASTER_PIP=\$(jq -r '.linux_vms[0].private_ip' pipe-${{ env.UNIQUE_ID }}.machines.json) && \
-            ALL_IPS=\$(jq -c '[.linux_vms[].private_ip]' pipe-${{ env.UNIQUE_ID }}.machines.json) && \
-            jq -n --argjson hosts \"\$ALL_IPS\" --arg master \"\$MASTER_PIP\" \
-              '{lme_cluster_mode: true, es_cluster_seed_hosts: \$hosts, es_master_publish_host: \$master}' \
-              > /tmp/cluster_vars.json && \
-            echo 'Cluster extra vars:' && cat /tmp/cluster_vars.json && \
-            scp /tmp/cluster_vars.json lme-user@${{ env.AZURE_IP }}:/tmp/cluster_vars.json
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} '
-              cd ~/LME && ansible-playbook ansible/site.yml -e @/tmp/cluster_vars.json
-            '
-          "
-
-      - name: Create cluster inventory and deploy to child nodes
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd ${INSTALLERS_DIR} && \
-            MACHINES_FILE=pipe-${{ env.UNIQUE_ID }}.machines.json && \
-            MASTER_PIP=\$(jq -r '.linux_vms[0].private_ip' \$MACHINES_FILE) && \
-            MASTER_PUB=\$(jq -r '.linux_vms[0].ip_address' \$MACHINES_FILE) && \
-            TMPFILE=\$(mktemp) && \
-            echo 'all:' > \$TMPFILE && \
-            echo '  vars:' >> \$TMPFILE && \
-            echo \"    es_master_host: \${MASTER_PIP}\" >> \$TMPFILE && \
-            echo '    es_cluster_seed_hosts:' >> \$TMPFILE && \
-            for ip in \$(jq -r '.linux_vms[].private_ip' \$MACHINES_FILE); do
-              echo \"      - \${ip}\" >> \$TMPFILE
-            done && \
-            echo '  children:' >> \$TMPFILE && \
-            echo '    elasticsearch:' >> \$TMPFILE && \
-            echo '      hosts:' >> \$TMPFILE && \
-            echo '        es1:' >> \$TMPFILE && \
-            echo \"          ansible_host: \${MASTER_PIP}\" >> \$TMPFILE && \
-            echo '          ansible_connection: local' >> \$TMPFILE && \
-            echo '          es_node_name: lme-elasticsearch' >> \$TMPFILE && \
-            echo '          es_is_initial_master: true' >> \$TMPFILE && \
-            echo \"          es_publish_host: \${MASTER_PIP}\" >> \$TMPFILE && \
-            i=2 && \
-            for ip in \$(jq -r '.linux_vms[1:][].private_ip' \$MACHINES_FILE); do
-              echo \"        es\${i}:\" >> \$TMPFILE && \
-              echo \"          ansible_host: \${ip}\" >> \$TMPFILE && \
-              echo '          ansible_user: lme-user' >> \$TMPFILE && \
-              echo \"          es_node_name: es\${i}\" >> \$TMPFILE && \
-              echo \"          es_publish_host: \${ip}\" >> \$TMPFILE
-              i=\$((i + 1))
-            done && \
-            echo 'Generated cluster inventory:' && cat \$TMPFILE && \
-            scp \$TMPFILE lme-user@\${MASTER_PUB}:~/LME/ansible/inventory/cluster.yml && \
-            rm \$TMPFILE
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} '
-              cd ~/LME && ansible-playbook -i ansible/inventory/cluster.yml ansible/elasticsearch.yml
-            '
-          "
-
-      - name: Verify cluster health
-        run: |
-          cd testing/v2/development
-          echo "Waiting for cluster to stabilize..."
-          sleep 120
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} 'sudo bash -c \"\
-              source /opt/lme/scripts/extract_secrets.sh -q && \
-              echo === Cluster Health === && \
-              curl -sk -u \\\"elastic:\\\$elastic\\\" https://localhost:9200/_cluster/health?pretty && \
-              echo === Cluster Nodes === && \
-              curl -sk -u \\\"elastic:\\\$elastic\\\" https://localhost:9200/_cat/nodes?v\"'
-          "
-
-      - name: Set up NFS server on master
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd ${INSTALLERS_DIR} && \
-            MACHINES_FILE=pipe-${{ env.UNIQUE_ID }}.machines.json && \
-            MASTER_PIP=\$(jq -r '.linux_vms[0].private_ip' \$MACHINES_FILE) && \
-            NFS_EXPORTS='/srv/es-snapshots' && \
-            for ip in \$(jq -r '.linux_vms[].private_ip' \$MACHINES_FILE); do
-              NFS_EXPORTS=\"\${NFS_EXPORTS} \${ip}(rw,sync,no_subtree_check,no_root_squash)\"
-            done && \
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-              \"sudo apt-get install -y nfs-kernel-server && \
-               sudo mkdir -p /srv/es-snapshots && sudo chmod 777 /srv/es-snapshots && \
-               echo '\${NFS_EXPORTS}' | sudo tee /etc/exports && \
-               sudo exportfs -ra && sudo systemctl start nfs-kernel-server\"
-          "
-
-      - name: Mount NFS on all nodes
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd ${INSTALLERS_DIR} && \
-            MACHINES_FILE=pipe-${{ env.UNIQUE_ID }}.machines.json && \
-            MASTER_PIP=\$(jq -r '.linux_vms[0].private_ip' \$MACHINES_FILE) && \
-            for pub_ip in \$(jq -r '.linux_vms[].ip_address' \$MACHINES_FILE); do
-              echo \"=== Mounting NFS on \${pub_ip} ===\" && \
-              ssh -o StrictHostKeyChecking=no lme-user@\${pub_ip} \
-                \"sudo apt-get install -y nfs-common && \
-                 sudo mkdir -p /mnt/es-snapshots && \
-                 sudo mount -t nfs \${MASTER_PIP}:/srv/es-snapshots /mnt/es-snapshots && \
-                 grep -q /mnt/es-snapshots /etc/fstab || echo '\${MASTER_PIP}:/srv/es-snapshots /mnt/es-snapshots nfs defaults 0 0' | sudo tee -a /etc/fstab\"
-            done
-          "
-
-      - name: Configure Elasticsearch NFS snapshot path on all nodes
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            cd ${INSTALLERS_DIR} && \
-            MACHINES_FILE=pipe-${{ env.UNIQUE_ID }}.machines.json && \
-            for pub_ip in \$(jq -r '.linux_vms[].ip_address' \$MACHINES_FILE); do
-              echo \"=== Configuring ES on \${pub_ip} ===\" && \
-              ssh -o StrictHostKeyChecking=no lme-user@\${pub_ip} '
-                # path.repo now set by Ansible template (elasticsearch.yml.j2) when lme_cluster_mode=true
-                # sudo grep -q /usr/share/elasticsearch/snapshots /opt/lme/config/elasticsearch.yml || \
-                #   sudo sed -i \"/\\/usr\\/share\\/elasticsearch\\/backups/a\\\\    - /usr/share/elasticsearch/snapshots\" /opt/lme/config/elasticsearch.yml
-                sudo mkdir -p /etc/containers/systemd/lme-elasticsearch.container.d/
-                echo \"[Container]
-          Volume=/mnt/es-snapshots:/usr/share/elasticsearch/snapshots\" | sudo tee /etc/containers/systemd/lme-elasticsearch.container.d/nfs-mount.conf
-                sudo systemctl daemon-reload && sudo systemctl restart lme-elasticsearch
-              '
-            done
-          "
-
-      - name: Verify NFS and cluster health after restart
-        run: |
-          cd testing/v2/development
-          INSTALLERS_DIR="/home/lme-user/LME/testing/v2/installers"
-          echo "Waiting for Elasticsearch to restart on all nodes..."
-          sleep 60
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            echo '=== Creating test file on master NFS export ===' && \
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-              'sudo touch /srv/es-snapshots/nfs_test_file' && \
-            cd ${INSTALLERS_DIR} && \
-            MACHINES_FILE=pipe-${{ env.UNIQUE_ID }}.machines.json && \
-            for pub_ip in \$(jq -r '.linux_vms[1:][].ip_address' \$MACHINES_FILE); do
-              echo \"=== Checking NFS on \${pub_ip} ===\" && \
-              ssh -o StrictHostKeyChecking=no lme-user@\${pub_ip} \
-                'df -h /mnt/es-snapshots && ls -la /mnt/es-snapshots/nfs_test_file'
-            done && \
-            echo '=== Cleaning up test file ===' && \
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-              'sudo rm /srv/es-snapshots/nfs_test_file'
-          "
-
-          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-            ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} 'sudo bash -c \"\
-              source /opt/lme/scripts/extract_secrets.sh -q && \
-              echo === Cluster Health After NFS === && \
-              curl -sk -u \\\"elastic:\\\$elastic\\\" https://localhost:9200/_cluster/health?pretty && \
-              echo === Cluster Nodes After NFS === && \
-              curl -sk -u \\\"elastic:\\\$elastic\\\" https://localhost:9200/_cat/nodes?v\"'
-          "
-
       - name: Retrieve Elastic password
         env:
           AZURE_IP: ${{ env.AZURE_IP }}