Skip to content

Commit a4b8209

Browse files
committed
test: Add e2e incident detection test
- e2e/incidents: 00.incidents.cy.ts test deploys pod that fails to start and creates a KubePodCrashLoop alert and checks that the alert later appears in the system. - views/incident-page: new function to find a specific alert within existing incident bars is added - support/commands.ts: Command for failing pod deployment + adding label that enables the incident detection to work properly during setup - fixtures/incidents/: yaml files for the failing pod deployment (plus custom alert rule)
1 parent a351aca commit a4b8209

8 files changed

Lines changed: 202 additions & 19 deletions

File tree

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
The test verifies the whole lifecycle of the Incident feature, without any external dependencies.
3+
*/
4+
import { commonPages } from '../../views/common';
5+
import { incidentsPage } from '../../views/incidents-page';
6+
7+
// Set constants for the operators that need to be installed for tests.
8+
const MCP = {
9+
namespace: 'openshift-cluster-observability-operator',
10+
packageName: 'cluster-observability-operator',
11+
operatorName: 'Cluster Observability Operator',
12+
config: {
13+
kind: 'UIPlugin',
14+
name: 'monitoring',
15+
},
16+
};
17+
18+
const MP = {
19+
namespace: 'openshift-monitoring',
20+
operatorName: 'Cluster Monitoring Operator',
21+
};
22+
23+
describe('Incidents', () => {
24+
before(() => {
25+
cy.afterBlockCOO(MCP, MP); // Following cypher best practices, the cleanup is done before the test block
26+
cy.beforeBlockCOO(MCP, MP);
27+
cy.createKubePodCrashLoopingAlert();
28+
});
29+
30+
after(() => {
31+
cy.afterBlockCOO(MCP, MP); // For compatibility with other tests
32+
});
33+
34+
it('Admin Perspective - Incidents tab renders and responds to interactions', () => {
35+
cy.log('1.1 Navigate to Alerting → Incidents');
36+
incidentsPage.goTo();
37+
commonPages.titleShouldHaveText('Incidents');
38+
});
39+
40+
it('Incident with KubePodCrashLooping alert is present in the alerts table', () => {
41+
incidentsPage.goTo();
42+
commonPages.titleShouldHaveText('Incidents');
43+
incidentsPage.clearAllFilters();
44+
45+
const intervalMs = 60_000;
46+
const maxMinutes = 20;
47+
48+
cy.waitUntil(() => incidentsPage.findIncidentWithAlert('KubePodCrashLooping'), {
49+
interval: intervalMs,
50+
timeout: maxMinutes * intervalMs
51+
});
52+
53+
incidentsPage
54+
.elements
55+
.alertsTable()
56+
.contains('KubePodCrashLooping')
57+
.should('exist');
58+
});
59+
});

web/cypress/e2e/incidents/01.incidents.cy.ts

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,16 @@
1+
/*
2+
The test verifies the basic functionality of the Incidents page and serves
3+
as a verification that the Incidents View is working as expected.
4+
5+
Currently, it depends on an alert being present in the cluster.
6+
In the future, mocking requests / injecting alerts should be considered.
7+
Natural creation of the alert is done in the 00.coo_incidents_e2e.cy.ts test,
8+
but takes significant time.
9+
*/
110

211
import { commonPages } from '../../views/common';
312
import { incidentsPage } from '../../views/incidents-page';
413

5-
// Set constants for the operators that need to be installed for tests.
614
const MCP = {
715
namespace: 'openshift-cluster-observability-operator',
816
packageName: 'cluster-observability-operator',
@@ -25,14 +33,15 @@ const ALERT_DESC = 'This is an alert meant to ensure that the entire alerting pi
2533
const ALERT_SUMMARY = 'An alert that should always be firing to certify that Alertmanager is working properly.'
2634
describe('Incidents', () => {
2735
before(() => {
36+
cy.afterBlockCOO(MCP, MP); // Following cypher best practices, the cleanup is done before the test block
2837
cy.beforeBlockCOO(MCP, MP);
29-
// TODO: Inject alerts into the database so the behavior is deterministic
3038
});
3139

3240
after(() => {
33-
cy.afterBlockCOO(MCP, MP);
41+
cy.afterBlockCOO(MCP, MP); // For compatibility with other tests
3442
});
3543

44+
3645
beforeEach(() => {
3746

3847
cy.log('Navigate to Observe → Incidents');
@@ -65,19 +74,13 @@ describe('Incidents', () => {
6574

6675
it('selecting an incident via chart shows alerts and adds groupId to URL', () => {
6776
incidentsPage.clearAllFilters();
68-
cy.pause();
6977
incidentsPage.selectIncidentByBarIndex(0);
70-
cy.pause();
7178
cy.url().should('match', /[?&]groupId=/);
72-
cy.pause();
7379
incidentsPage.elements.alertsChartSvg().find('path').should('exist');
74-
cy.pause();
7580
});
7681

7782
it('shows alerts table and allows expanding a row', () => {
78-
cy.pause();
7983
incidentsPage.elements.alertsTable().should('exist');
8084
incidentsPage.expandRow(0);
81-
cy.pause();
8285
});
8386
});
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: crash-loop
5+
namespace: openshift-monitoring
6+
spec:
7+
replicas: 1
8+
selector:
9+
matchLabels:
10+
app: crash-loop
11+
template:
12+
metadata:
13+
labels:
14+
app: crash-loop
15+
spec:
16+
containers:
17+
- name: crash-loop
18+
image: busybox
19+
command: ["sh", "-c", "exit 1"] # Exit immediately with a failure
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: PrometheusRule
3+
metadata:
4+
labels:
5+
app.kubernetes.io/name: kube-prometheus
6+
app.kubernetes.io/part-of: openshift-monitoring
7+
prometheus: k8s
8+
role: alert-rules
9+
name: kubernetes-monitoring-podcrash-rules
10+
namespace: openshift-monitoring
11+
spec:
12+
groups:
13+
- name: kubernetes-apps
14+
rules:
15+
- alert: KubePodCrashLooping
16+
annotations:
17+
description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
18+
}}) is in waiting state (reason: "CrashLoopBackOff").'
19+
summary: Pod is crash looping.
20+
expr: |
21+
max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", namespace=~"(openshift-.*|kube-.*|default)",job="kube-state-metrics"}[5m]) >= 1
22+
for: 5m
23+
labels:
24+
severity: warning

web/cypress/support/commands.ts

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import Shadow = Cypress.Shadow;
66
import 'cypress-wait-until';
77
import { guidedTour } from '../views/tour';
88
import { nav } from '../views/nav';
9+
import './nav';
910
import { operatorHubPage } from '../views/operator-hub-page';
1011

1112

@@ -42,11 +43,6 @@ declare global {
4243
bySemanticElement(element: string, text?: string): Chainable<JQuery<HTMLElement>>;
4344
byAriaLabel(label: string, options?: Partial<Loggable & Timeoutable & Withinable & Shadow>): Chainable<JQuery<HTMLElement>>;
4445
byPFRole(role: string, options?: Partial<Loggable & Timeoutable & Withinable & Shadow>): Chainable<JQuery<HTMLElement>>;
45-
}
46-
}
47-
48-
declare global {
49-
interface Chainable {
5046
switchPerspective(perspective: string);
5147
uiLogin(provider: string, username: string, password: string);
5248
uiLogout();
@@ -546,6 +542,7 @@ Cypress.Commands.add('beforeBlockCOO', (MCP: { namespace: string, operatorName:
546542
expect(result.code).to.eq(0);
547543
cy.log(`Monitoring plugin pod is now running in namespace: ${MCP.namespace}`);
548544
});
545+
cy.exec(`oc label namespace openshift-cluster-observability-operator openshift.io/cluster-monitoring="true" --kubeconfig ${Cypress.env('KUBECONFIG_PATH')}`)
549546
//TODO: https://issues.redhat.com/browse/OCPBUGS-58468 - console reload and logout was happening more often
550547
// cy.get('.pf-v5-c-alert, .pf-v6-c-alert', { timeout: readyTimeoutMilliseconds })
551548
// .contains('Web console update is available')
@@ -612,6 +609,8 @@ Cypress.Commands.add('afterBlockCOO', (MCP: { namespace: string, operatorName: s
612609
`oc adm policy remove-cluster-role-from-user cluster-admin ${Cypress.env('LOGIN_USERNAME')} --kubeconfig ${Cypress.env('KUBECONFIG_PATH')}`,
613610
);
614611

612+
cy.executeAndDelete(`oc label namespace openshift-cluster-observability-operator openshift.io/cluster-monitoring- --kubeconfig ${Cypress.env('KUBECONFIG_PATH')}`)
613+
615614
//TODO: https://issues.redhat.com/browse/OCPBUGS-58468 - console reload and logout was happening more often
616615
// cy.get('.pf-v5-c-alert, .pf-v6-c-alert', { timeout: 120000 })
617616
// .contains('Web console update is available')
@@ -626,4 +625,26 @@ Cypress.Commands.add('afterBlockCOO', (MCP: { namespace: string, operatorName: s
626625

627626
}
628627
cy.log('After block COO completed');
628+
});
629+
630+
// Apply incident fixture manifests to the cluster
631+
Cypress.Commands.add('createKubePodCrashLoopingAlert', () => {
632+
const kubeconfigPath = Cypress.env('KUBECONFIG_PATH');
633+
cy.exec(
634+
`oc apply -f ./cypress/fixtures/incidents/prometheus_rule_pod_crash_loop.yaml --kubeconfig ${kubeconfigPath}`,
635+
);
636+
cy.exec(
637+
`oc apply -f ./cypress/fixtures/incidents/pod_crash_loop.yaml --kubeconfig ${kubeconfigPath}`,
638+
);
639+
});
640+
641+
// Clean up incident fixture manifests from the cluster
642+
Cypress.Commands.add('cleanupIncidentsFixtures', () => {
643+
const kubeconfigPath = Cypress.env('KUBECONFIG_PATH');
644+
cy.executeAndDelete(
645+
`oc delete -f ./cypress/fixtures/incidents/pod_crash_loop.yaml --ignore-not-found=true --kubeconfig ${kubeconfigPath}`,
646+
);
647+
cy.executeAndDelete(
648+
`oc delete -f ./cypress/fixtures/incidents/prometheus_rule_pod_crash_loop.yaml --ignore-not-found=true --kubeconfig ${kubeconfigPath}`,
649+
);
629650
});

web/cypress/support/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import './nav';
22
import './selectors';
33
import './commands';
4+
import './alert-injection';
5+
import './prometheus-query-mocks';
46

57
export const checkErrors = () =>
68
cy.window().then((win) => {

web/cypress/tsconfig.json

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,8 @@
44
"lib": ["es5", "dom"],
55
"types": ["cypress", "node"]
66
},
7-
"include": ["**/*.ts"]
7+
8+
"include": [
9+
"**/*.ts"
10+
]
811
}

web/cypress/views/incidents-page.ts

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,18 @@ export const incidentsPage = {
3333

3434
selectIncidentByBarIndex: (index = 0) => {
3535
cy.log('incidentsPage.selectIncidentByBarIndex');
36-
incidentsPage.elements.incidentsChartCard()
37-
.find('path[role="presentation"]')
38-
.eq(index)
39-
.click({ force: true });
36+
// TODO: There are two bars for the same incident, one transparent, one not.
37+
// Right now, we use both which works but doubles the execution time.
38+
return cy.waitUntil(() => {
39+
return incidentsPage.elements.incidentsChartCard()
40+
.find('path[role="presentation"]')
41+
.eq(index)
42+
.click({ force: true })
43+
.then(() => {
44+
return Cypress.$('[aria-label="alerts-table"]').length > 0;
45+
});
46+
}, { interval: 10000, timeout: 120000 })
47+
.then(() => incidentsPage.elements.alertsTable().scrollIntoView().should('be.visible'))
4048
},
4149

4250
expandRow: (rowIndex = 0) => {
@@ -51,6 +59,50 @@ export const incidentsPage = {
5159
});
5260
},
5361

62+
63+
findIncidentWithAlert: (alertName: string): Cypress.Chainable<boolean> => {
64+
cy.log(`incidentsPage.findIncidentWithAlert: ${alertName}`);
65+
return incidentsPage.elements.incidentsChartCard()
66+
.find('path[role="presentation"]')
67+
.then(($bars) => {
68+
const totalBars = $bars.length;
69+
if (totalBars <= 3) { // 3 paths are always present in the legend,
70+
// their parent is g without presentation label opposed ot the proper bars
71+
cy.task('log', 'No bars found in incidents chart');
72+
return cy.wrap(false);
73+
}
74+
75+
const tryIndex = (index: number): Cypress.Chainable<boolean> => {
76+
if (index >= totalBars - 3) {
77+
return cy.wrap(false);
78+
}
79+
80+
return cy
81+
.wrap(null)
82+
.then(() => {
83+
incidentsPage.selectIncidentByBarIndex(index);
84+
return null;
85+
})
86+
.then(() => incidentsPage.elements.alertsTable().invoke('text'))
87+
.then((text) => {
88+
if (String(text).includes(alertName)) {
89+
return cy.wrap(true);
90+
}
91+
// Expand a row if present to surface nested details
92+
incidentsPage.expandRow(0);
93+
return incidentsPage.elements.alertsTable().invoke('text').then((text2) => {
94+
if (String(text2).includes(alertName)) {
95+
return cy.wrap(true);
96+
}
97+
return tryIndex(index + 1);
98+
});
99+
});
100+
};
101+
102+
return tryIndex(0);
103+
});
104+
},
105+
54106
// Centralized element selectors - all selectors defined in one place
55107
elements: {
56108
// Page structure

0 commit comments

Comments
 (0)