Skip to content

Commit 8093ebf

Browse files
gkechgkech
authored
K8SPSMDB-1363 vault e2e test - fix vault file access mode (#2279)
* K8SPSMDB-1363 vault e2e test and fix for chmod 0600 * add test to pipelines * handle container not ready * fixes * adjust the logic for the mode of the mount file
1 parent b58045d commit 8093ebf

12 files changed

Lines changed: 391 additions & 14 deletions

File tree

‎e2e-tests/demand-backup-snapshot-vault/compare/find.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
switched to db myApp
2+
{ "_id" : , "x" : 100500 }
3+
bye

‎e2e-tests/demand-backup-snapshot-vault/conf/backup.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: psmdb.percona.com/v1
2+
kind: PerconaServerMongoDBBackup
3+
metadata:
4+
finalizers:
5+
- percona.com/delete-backup
6+
name:
7+
spec:
8+
type: external
9+
clusterName: some-name
10+
volumeSnapshotClass:

‎e2e-tests/demand-backup-snapshot-vault/conf/restore.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: psmdb.percona.com/v1
2+
kind: PerconaServerMongoDBRestore
3+
metadata:
4+
name:
5+
spec:
6+
clusterName: some-name
7+
backupName:

‎e2e-tests/demand-backup-snapshot-vault/conf/role-binding.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: RoleBinding
3+
metadata:
4+
name: vault-role-binding
5+
namespace: <namespace>
6+
roleRef:
7+
apiGroup: rbac.authorization.k8s.io
8+
kind: ClusterRole
9+
name: system:auth-delegator
10+
subjects:
11+
- kind: ServiceAccount
12+
name: percona-server-mongodb-operator
13+
namespace: <namespace>

‎e2e-tests/demand-backup-snapshot-vault/conf/secrets.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
kind: Secret
3+
metadata:
4+
name: some-users
5+
type: Opaque
6+
data:
7+
MONGODB_BACKUP_USER: YmFja3VwJCMl
8+
MONGODB_BACKUP_PASSWORD: YmFja3VwMTIzNDU2Iw==
9+
MONGODB_DATABASE_ADMIN_USER: ZGF0YWJhc2VBZG1pbg==
10+
MONGODB_DATABASE_ADMIN_PASSWORD: ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
11+
MONGODB_USER_ADMIN_USER: dXNlckFkbWlu
12+
MONGODB_USER_ADMIN_PASSWORD: dXNlckFkbWluMTIzNDU2
13+
MONGODB_CLUSTER_ADMIN_USER: Y2x1c3RlckFkbWlu
14+
MONGODB_CLUSTER_ADMIN_PASSWORD: Y2x1c3RlckFkbWluMTIzNDU2
15+
MONGODB_CLUSTER_MONITOR_USER: Y2x1c3Rlck1vbml0b3I=
16+
MONGODB_CLUSTER_MONITOR_PASSWORD: Y2x1c3Rlck1vbml0b3IxMjM0NTY=

‎e2e-tests/demand-backup-snapshot-vault/conf/some-name.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
apiVersion: psmdb.percona.com/v1
2+
kind: PerconaServerMongoDB
3+
metadata:
4+
finalizers:
5+
- percona.com/delete-psmdb-pvc
6+
name: some-name
7+
spec:
8+
#platform: openshift
9+
image:
10+
imagePullPolicy: Always
11+
updateStrategy: SmartUpdate
12+
tls:
13+
mode: requireTLS
14+
backup:
15+
enabled: true
16+
image: perconalab/percona-server-mongodb-operator:main-backup
17+
storages:
18+
minio:
19+
type: s3
20+
s3:
21+
credentialsSecret: minio-secret
22+
region: us-east-1
23+
bucket: operator-testing
24+
endpointUrl: http://minio-service:9000/
25+
insecureSkipTLSVerify: false
26+
vault:
27+
endpointURL: http://vault-service.NAME_SPACE.svc.cluster.local:8200
28+
syncUsers:
29+
role: operator
30+
tokenSecret: vault-sync-secret
31+
replsets:
32+
- name: rs0
33+
affinity:
34+
antiAffinityTopologyKey: none
35+
resources:
36+
limits:
37+
cpu: 500m
38+
memory: 1G
39+
requests:
40+
cpu: 100m
41+
memory: 0.1G
42+
volumeSpec:
43+
persistentVolumeClaim:
44+
resources:
45+
requests:
46+
storage: 3Gi
47+
expose:
48+
enabled: false
49+
type: ClusterIP
50+
size: 3
51+
configuration: |
52+
operationProfiling:
53+
mode: slowOp
54+
slowOpThresholdMs: 100
55+
security:
56+
enableEncryption: true
57+
vault:
58+
serverName: vault-service
59+
port: 8200
60+
tokenFile: /etc/mongodb-vault/token
61+
secret: secret/data/some-name/rs0
62+
disableTLSForTesting: true
63+
setParameter:
64+
ttlMonitorSleepSecs: 60
65+
wiredTigerConcurrentReadTransactions: 128
66+
wiredTigerConcurrentWriteTransactions: 128
67+
storage:
68+
engine: wiredTiger
69+
wiredTiger:
70+
collectionConfig:
71+
blockCompressor: snappy
72+
engineConfig:
73+
directoryForIndexes: false
74+
journalCompressor: snappy
75+
indexConfig:
76+
prefixCompression: true
77+
secrets:
78+
users: some-users
79+
vault: vault-secret

‎e2e-tests/demand-backup-snapshot-vault/run‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
#!/bin/bash
2+
3+
set -o errexit
4+
5+
test_dir=$(realpath "$(dirname "$0")")
6+
. "${test_dir}/../functions"
7+
set_debug
8+
9+
vault_name="vault-service"
10+
11+
VOLUME_SNAPSHOT_CLASS=$(deploy_volume_snapshot_class)
12+
13+
create_tls_secret() {
14+
local name=$1
15+
local service_name=$2
16+
17+
local tmp_dir
18+
tmp_dir=$(mktemp -d)
19+
20+
openssl genrsa -out "${tmp_dir}/vault.key" 2048
21+
22+
cat <<EOF >"${tmp_dir}/csr.conf"
23+
[req]
24+
distinguished_name = req_distinguished_name
25+
x509_extensions = v3_req
26+
prompt = no
27+
[req_distinguished_name]
28+
CN = ${service_name}.${namespace}.svc
29+
[v3_req]
30+
basicConstraints = CA:FALSE
31+
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
32+
extendedKeyUsage = serverAuth
33+
subjectAltName = @alt_names
34+
[alt_names]
35+
DNS.1 = ${service_name}
36+
DNS.2 = *.${service_name}
37+
DNS.3 = *.${service_name}.${namespace}
38+
DNS.4 = *.${service_name}.${namespace}.svc
39+
DNS.5 = ${service_name}.${namespace}.svc
40+
DNS.6 = ${service_name}.${namespace}.svc.cluster.local
41+
DNS.7 = *.${service_name}.${namespace}.svc.cluster.local
42+
IP.1 = 127.0.0.1
43+
EOF
44+
openssl req -x509 -new -nodes \
45+
-key "${tmp_dir}"/vault.key \
46+
-sha256 \
47+
-days 3650 \
48+
-out "${tmp_dir}"/vault.crt \
49+
-config "${tmp_dir}"/csr.conf \
50+
-extensions v3_req
51+
52+
if [ ! -s "${tmp_dir}"/vault.crt ]; then
53+
echo "failed to generate vault tls certificate"
54+
exit 1
55+
fi
56+
57+
cp "${tmp_dir}"/vault.crt "${tmp_dir}"/vault.ca
58+
59+
kubectl create secret generic "$name" \
60+
--from-file=tls.key="${tmp_dir}"/vault.key \
61+
--from-file=tls.crt="${tmp_dir}"/vault.crt \
62+
--from-file=ca.crt="${tmp_dir}"/vault.ca
63+
64+
rm -rf "$tmp_dir"
65+
}
66+
67+
setup_vault() {
68+
local sa_namespace=$namespace
69+
if [ -n "$OPERATOR_NS" ]; then
70+
sa_namespace=$OPERATOR_NS
71+
fi
72+
73+
deploy_vault $vault_name \
74+
--set "global.enabled=true" \
75+
--set "global.tlsDisable=true" \
76+
\
77+
--set "server.standalone.enabled=true"
78+
sleep 10
79+
80+
wait_pod $vault_name-0
81+
82+
sleep 20
83+
84+
kubectl_bin exec $vault_name-0 -- vault auth enable kubernetes
85+
cat "$test_dir"/conf/role-binding.yml \
86+
| yq ".metadata.namespace=\"$namespace\"" \
87+
| yq ".subjects[0].namespace=\"$sa_namespace\"" \
88+
| kubectl_bin apply -f -
89+
90+
token=$(kubectl_bin exec $vault_name-0 -- vault token create -policy=operator -format=json | jq -r '.auth.client_token')
91+
kubectl_bin create secret generic vault-sync-secret --from-literal=token="${token}"
92+
93+
# shellcheck disable=SC2016
94+
kubectl_bin exec $vault_name-0 -- sh -c 'vault write auth/kubernetes/config kubernetes_host=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT'
95+
96+
kubectl_bin exec $vault_name-0 -- sh -c 'vault policy write operator - <<EOF
97+
# KVv1
98+
path "secret/psmdb/operator/*" {
99+
capabilities = ["read"]
100+
}
101+
102+
# KVv2
103+
path "secret/data/psmdb/operator/*" {
104+
capabilities = ["read"]
105+
}
106+
EOF
107+
'
108+
109+
kubectl_bin exec $vault_name-0 -- vault write auth/kubernetes/role/operator \
110+
bound_service_account_names=percona-server-mongodb-operator \
111+
bound_service_account_namespaces="$sa_namespace" \
112+
policies=operator \
113+
ttl=1h
114+
}
115+
116+
vault_append() {
117+
local key=$1
118+
local value=$2
119+
120+
local tmp_json
121+
tmp_json=$(mktemp)
122+
local new_tmp_json
123+
new_tmp_json=$(mktemp)
124+
125+
kubectl_bin exec $vault_name-0 -- sh -c "vault kv get -format=json -mount=secret psmdb/operator/$namespace/$cluster/users" | jq '.data.data' >"$tmp_json"
126+
127+
if [ ! -s "$tmp_json" ]; then
128+
echo '{}' >"$tmp_json"
129+
fi
130+
131+
jq --arg key "$key" --arg value "$value" \
132+
'(. // {}) + {($key): $value}' \
133+
"$tmp_json" >"$new_tmp_json"
134+
135+
kubectl_bin cp "$new_tmp_json" $vault_name-0:/tmp/data_new.json
136+
kubectl_bin exec $vault_name-0 -- sh -c "vault kv put -mount=secret psmdb/operator/$namespace/$cluster/users @\"/tmp/data_new.json\""
137+
138+
rm -f "$tmp_json" "$new_tmp_json"
139+
}
140+
141+
run_snapshot_backup() {
142+
local backup_name=$1
143+
144+
log "running snapshot backup $backup_name"
145+
146+
yq eval '.metadata.name = "'${backup_name}'" | .spec.volumeSnapshotClass = "'${VOLUME_SNAPSHOT_CLASS}'"' \
147+
"${test_dir}/conf/backup.yml" \
148+
| kubectl_bin apply -f -
149+
}
150+
151+
run_snapshot_recovery_check() {
152+
local backup_name=$1
153+
154+
wait_restore "${backup_name}" "${cluster}" "ready" "0" "3000"
155+
156+
if [ "$(kubectl_bin get psmdb "${cluster}" -o jsonpath='{.metadata.annotations.percona\.com/resync-pbm}')" != "true" ]; then
157+
echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a snapshot restore"
158+
exit 1
159+
fi
160+
161+
wait_cluster_consistency "${cluster}"
162+
wait_for_pbm_operations "${cluster}"
163+
164+
compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-0.${cluster}-rs0.${namespace}" "" "" "" "" "" "true"
165+
compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-1.${cluster}-rs0.${namespace}" "" "" "" "" "" "true"
166+
compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-2.${cluster}-rs0.${namespace}" "" "" "" "" "" "true"
167+
}
168+
169+
cluster="some-name"
170+
171+
create_infra "${namespace}"
172+
deploy_minio
173+
apply_s3_storage_secrets
174+
175+
desc 'Setting up Vault'
176+
setup_vault
177+
178+
desc 'Seeding initial credentials in Vault'
179+
vault_append "MONGODB_USER_ADMIN_PASSWORD" "userAdmin123456"
180+
vault_append "MONGODB_BACKUP_PASSWORD" "backup123456#"
181+
182+
desc 'Deploying PSMDB cluster with Vault and snapshot backup'
183+
kubectl_bin apply -f "${test_dir}/conf/secrets.yml"
184+
apply_cluster "${test_dir}/conf/${cluster}.yml"
185+
kubectl_bin apply -f "${conf_dir}/client_with_tls.yml"
186+
187+
echo 'check if all pods started'
188+
wait_for_running "${cluster}-rs0" 3
189+
wait_cluster_consistency "${cluster}"
190+
191+
sleep 60 # give time for resync to start
192+
wait_for_pbm_operations "${cluster}"
193+
194+
desc 'Writing test data'
195+
run_mongo_tls \
196+
'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \
197+
"userAdmin:userAdmin123456@${cluster}-rs0.${namespace}"
198+
sleep 1
199+
run_mongo_tls \
200+
'use myApp\n db.test.insert({ x: 100500 })' \
201+
"myApp:myPass@${cluster}-rs0.${namespace}"
202+
sleep 5
203+
compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-0.${cluster}-rs0.${namespace}" "" "" "" "" "" "true"
204+
compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-1.${cluster}-rs0.${namespace}" "" "" "" "" "" "true"
205+
compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-2.${cluster}-rs0.${namespace}" "" "" "" "" "" "true"
206+
207+
desc 'Running snapshot backup'
208+
backup_name="backup-snapshot-vault"
209+
run_snapshot_backup "${backup_name}"
210+
wait_backup "${backup_name}"
211+
212+
desc 'Drop collection and restore from snapshot'
213+
run_mongo_tls 'use myApp\n db.test.drop()' "myApp:myPass@${cluster}-rs0.${namespace}"
214+
run_restore "${backup_name}"
215+
run_snapshot_recovery_check "${backup_name}"
216+
217+
desc 'Verifying Vault sync still works after snapshot restore'
218+
newpass="vault-test-password-after-restore"
219+
vault_append "MONGODB_BACKUP_PASSWORD" "$newpass"
220+
sleep 25
221+
wait_cluster_consistency "${cluster}"
222+
sleep 15
223+
backup_user=$(getUserData "some-users" "MONGODB_BACKUP_USER")
224+
ping=$(run_mongo_tls "db.runCommand({ ping: 1 }).ok" "$backup_user:$newpass@${cluster}-rs0-0.${cluster}-rs0.${namespace}" "mongodb" "" "--quiet")
225+
if [ "${ping}" != "1" ]; then
226+
echo "Vault sync check failed: could not authenticate as $backup_user with new password"
227+
exit 1
228+
fi
229+
230+
log "Vault sync after restore: OK"
231+
232+
destroy_vault "${vault_name}"
233+
destroy "${namespace}"
234+
235+
desc 'test passed'

‎e2e-tests/run-backups.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ demand-backup-physical-minio-native-tls
2929
demand-backup-physical-sharded-minio-native
3030
demand-backup-sharded
3131
demand-backup-snapshot
32+
demand-backup-snapshot-vault
3233
pitr-physical
3334
pitr-physical-backup-source
3435
pitr-sharded

‎e2e-tests/run-distro.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ demand-backup-physical-sharded-azure
3030
demand-backup-physical-sharded-gcp-native
3131
demand-backup-physical-sharded-minio
3232
demand-backup-sharded
33+
demand-backup-snapshot-vault
3334
init-deploy
3435
ldap
3536
ldap-tls

‎e2e-tests/run-pr.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ demand-backup-physical-sharded-minio
3737
demand-backup-physical-sharded-minio-native
3838
demand-backup-sharded
3939
demand-backup-snapshot
40+
demand-backup-snapshot-vault
4041
disabled-auth
4142
expose-sharded
4243
finalizer

0 commit comments

Comments
 (0)