Merge pull request #509 from DavidRajnoha/incident-detection-tests

OBSINTA-776: Incident detection tests

Author: openshift-merge-bot[bot]

web/cypress/e2e/incidents/00.coo_incidents_e2e.cy.ts

@@ -0,0 +1,70 @@
+/*
+The test verifies the whole lifecycle of the Incident feature, without any external dependencies.
+The run time can be 15 - 20 minutes. (Waiting untill the incident detection captures the new alert)
+*/
+import { commonPages } from '../../views/common';
+import { incidentsPage } from '../../views/incidents-page';
+
+// Set constants for the operators that need to be installed for tests.
+const MCP = {
+  namespace: 'openshift-cluster-observability-operator',
+  packageName: 'cluster-observability-operator',
+  operatorName: 'Cluster Observability Operator',
+  config: {
+    kind: 'UIPlugin',
+    name: 'monitoring',
+  },
+};
+
+const MP = {
+  namespace: 'openshift-monitoring',
+  operatorName: 'Cluster Monitoring Operator',
+};
+
+describe('BVT: Incidents - e2e', () => {
+  let currentAlertName: string;
+
+  before(() => {
+    cy.afterBlockCOO(MCP, MP); // Following cypher best practices, the cleanup is done before the test block
+    cy.beforeBlockCOO(MCP, MP);
+    
+    cy.cleanupIncidentPrometheusRules(); 
+
+    // Create the alert and capture the random name
+    cy.createKubePodCrashLoopingAlert().then((alertName) => {
+      currentAlertName = alertName;
+      cy.log(`Test will look for alert: ${currentAlertName}`);
+    });
+  });
+
+  after(() => {
+    cy.afterBlockCOO(MCP, MP); // For compatibility with other tests
+  });
+
+  it('1. Admin perspective - Incidents page - Incident with custom alert lifecycle', () => {
+    cy.log('1.1 Navigate to Incidents page and clear filters');
+    incidentsPage.goTo();
+    commonPages.titleShouldHaveText('Incidents');
+    incidentsPage.clearAllFilters();
+    
+    const intervalMs = 60_000;
+    const maxMinutes = 30; 
+
+    cy.log('1.2 Wait for incident with custom alert to appear');
+    cy.waitUntilWithCustomTimeout(
+      () => incidentsPage.findIncidentWithAlert(currentAlertName),
+      { 
+        interval: intervalMs, 
+        timeout: maxMinutes * intervalMs,
+        timeoutMessage: `Custom timeout: Incident with alert "${currentAlertName}" did not appear within ${maxMinutes} minutes.`
+      }
+    );
+
+    cy.log('1.3 Verify custom alert appears in alerts table');
+    incidentsPage
+      .elements
+      .alertsTable()
+      .contains(currentAlertName)
+      .should('exist');
+  });
+});

web/cypress/e2e/incidents/01.incidents.cy.ts

@@ -0,0 +1,102 @@
+/*
+The test verifies the basic functionality of the Incidents page and serves
+as a verification that the Incidents View is working as expected.
+
+Currently, it depends on an alert being present in the cluster.
+In the future, mocking requests / injecting alerts should be considered.
+Natural creation of the alert is done in the 00.coo_incidents_e2e.cy.ts test, 
+but takes significant time.
+*/
+
+import { commonPages } from '../../views/common';
+import { incidentsPage } from '../../views/incidents-page';
+
+const MCP = {
+  namespace: 'openshift-cluster-observability-operator',
+  packageName: 'cluster-observability-operator',
+  operatorName: 'Cluster Observability Operator',
+  config: {
+    kind: 'UIPlugin',
+    name: 'monitoring',
+  },
+};
+
+const MP = {
+  namespace: 'openshift-monitoring',
+  operatorName: 'Cluster Monitoring Operator',
+};
+
+const ALERTNAME = 'Watchdog';
+const NAMESPACE = 'openshift-monitoring';
+const SEVERITY = 'Critical';
+const ALERT_DESC = 'This is an alert meant to ensure that the entire alerting pipeline is functional. This alert is always firing, therefore it should always be firing in Alertmanager and always fire against a receiver. There are integrations with various notification mechanisms that send a notification when this alert is not firing. For example the "DeadMansSnitch" integration in PagerDuty.'
+const ALERT_SUMMARY = 'An alert that should always be firing to certify that Alertmanager is working properly.'
+describe('BVT: Incidents - UI', () => {
+  before(() => {
+    cy.afterBlockCOO(MCP, MP); // Following cypher best practices, the cleanup is done before the test block
+    cy.beforeBlockCOO(MCP, MP);
+  });
+
+  after(() => {
+    cy.afterBlockCOO(MCP, MP); // For compatibility with other tests
+  });
+
+
+  beforeEach(() => {
+
+    cy.log('Navigate to Observe → Incidents');
+    incidentsPage.goTo();
+    commonPages.titleShouldHaveText('Incidents');
+  });
+
+  it('1. Admin perspective - Incidents page - Toolbar and charts toggle functionality', () => {
+    cy.log('1.1 Verify toolbar and toggle charts button');
+    incidentsPage.elements.toolbar().should('be.visible');
+    incidentsPage.elements.toggleChartsButton().should('be.visible');
+    incidentsPage.elements.toggleChartsButton().click();
+    
+    cy.log('1.2 Verify charts are hidden after toggle');
+    incidentsPage.elements.incidentsChartTitle().should('not.exist');
+    incidentsPage.elements.alertsChartTitle().should('not.exist');
+    incidentsPage.elements.toggleChartsButton().click();
+  });
+
+  it('2. Admin perspective - Incidents page - Days filter functionality', () => {
+    cy.log('2.1 Set days filter to 3 days');
+    incidentsPage.setDays('3 days');
+    
+    cy.log('2.2 Verify filter and URL update');
+    incidentsPage.elements.daysSelect().should('contain.text', '3 days');
+    cy.url().should('match', /[?&]days=3\+days/);
+  });
+
+  it('3. Admin perspective - Incidents page - Critical filter functionality', () => {
+    cy.log('3.1 Clear filters and toggle Critical filter');
+    incidentsPage.clearAllFilters();
+    incidentsPage.toggleFilter('Critical');
+    
+    cy.log('3.2 Verify URL is updated with incident filters');
+    cy.url().should('match', /incidentFilters=.*Critical/);
+  });
+
+  it('4. Admin perspective - Incidents page - Charts and alerts empty state', () => {
+    cy.log('4.1 Verify chart titles are visible');
+    incidentsPage.elements.incidentsChartTitle().should('be.visible');
+    incidentsPage.elements.alertsChartTitle().should('be.visible');
+    
+    cy.log('4.2 Verify alerts chart shows empty state');
+    incidentsPage.elements.alertsChartEmptyState().should('exist');
+  });
+
+  it('5. Admin perspective - Incidents page - Incident selection and alert details', () => {
+    cy.log('5.1 Select incident and verify alert details');
+    incidentsPage.clearAllFilters();
+    incidentsPage.selectIncidentByBarIndex(0);
+    cy.url().should('match', /[?&]groupId=/);
+    incidentsPage.elements.alertsChartSvg().find('path').should('exist');
+
+    cy.log('5.2 Verify alerts table and expand first row');
+    incidentsPage.elements.alertsTable().should('exist');
+    incidentsPage.expandRow(0);
+  });
+});

web/cypress/fixtures/incidents/pod_crash_loop.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: crash-loop
+ namespace: openshift-monitoring
+spec:
+ replicas: 1
+ selector:
+   matchLabels:
+     app: crash-loop
+ template:
+   metadata:
+     labels:
+       app: crash-loop
+   spec:
+     containers:
+     - name: crash-loop
+       image: busybox
+       command: ["sh", "-c", "exit 1"]  # Exit immediately with a failure

web/cypress/fixtures/incidents/prometheus_rule_pod_crash_loop.yaml

@@ -0,0 +1,24 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+ labels:
+   app.kubernetes.io/name: kube-prometheus
+   app.kubernetes.io/part-of: openshift-monitoring
+   prometheus: k8s
+   role: alert-rules
+ name: kubernetes-monitoring-podcrash-rules
+ namespace: openshift-monitoring
+spec:
+ groups:
+ - name: kubernetes-apps
+   rules:
+   - alert: {{ALERT_NAME}}
+     annotations:
+       description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
+         }}) is in waiting state (reason: "CrashLoopBackOff").'
+       summary: Pod is crash looping.
+     expr: |
+       max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", namespace=~"(openshift-.*|kube-.*|default)",job="kube-state-metrics"}[5m]) >= 1
+     for: 5m
+     labels:
+       severity: warning

web/cypress/support/commands.ts

@@ -6,6 +6,7 @@ import Shadow = Cypress.Shadow;
 import 'cypress-wait-until';
 import { guidedTour } from '../views/tour';
 import { nav } from '../views/nav';
+import './nav';
 import { operatorHubPage } from '../views/operator-hub-page';
 
 
@@ -42,11 +43,6 @@ declare global {
     bySemanticElement(element: string, text?: string): Chainable<JQuery<HTMLElement>>;
     byAriaLabel(label: string, options?: Partial<Loggable & Timeoutable & Withinable & Shadow>): Chainable<JQuery<HTMLElement>>;
     byPFRole(role: string, options?: Partial<Loggable & Timeoutable & Withinable & Shadow>): Chainable<JQuery<HTMLElement>>;
-  }
-}
-
-declare global {
-  interface Chainable {
     switchPerspective(perspective: string);
     uiLogin(provider: string, username: string, password: string);
     uiLogout();
@@ -55,6 +51,10 @@ declare global {
     adminCLI(command: string, options?);
     login(provider?: string, username?: string, password?: string): Chainable<Element>;
     executeAndDelete(command: string);
+    waitUntilWithCustomTimeout(
+      fn: () => any,
+      options: { interval: number; timeout: number; timeoutMessage: string }
+    ): Cypress.Chainable<any>;
   }
 }
 
@@ -546,6 +546,7 @@ Cypress.Commands.add('beforeBlockCOO', (MCP: { namespace: string, operatorName:
     expect(result.code).to.eq(0);
     cy.log(`Monitoring plugin pod is now running in namespace: ${MCP.namespace}`);
   });
+  cy.exec(`oc label namespace openshift-cluster-observability-operator openshift.io/cluster-monitoring="true" --kubeconfig ${Cypress.env('KUBECONFIG_PATH')}`)
   //TODO: https://issues.redhat.com/browse/OCPBUGS-58468 - console reload and logout was happening more often
   // cy.get('.pf-v5-c-alert, .pf-v6-c-alert', { timeout: readyTimeoutMilliseconds })
   //   .contains('Web console update is available')
@@ -612,6 +613,8 @@ Cypress.Commands.add('afterBlockCOO', (MCP: { namespace: string, operatorName: s
       `oc adm policy remove-cluster-role-from-user cluster-admin ${Cypress.env('LOGIN_USERNAME')} --kubeconfig ${Cypress.env('KUBECONFIG_PATH')}`,
     );
 
+    cy.executeAndDelete(`oc label namespace openshift-cluster-observability-operator openshift.io/cluster-monitoring- --kubeconfig ${Cypress.env('KUBECONFIG_PATH')}`)
+
     //TODO: https://issues.redhat.com/browse/OCPBUGS-58468 - console reload and logout was happening more often
     // cy.get('.pf-v5-c-alert, .pf-v6-c-alert', { timeout: 120000 })
     //   .contains('Web console update is available')
@@ -626,4 +629,83 @@ Cypress.Commands.add('afterBlockCOO', (MCP: { namespace: string, operatorName: s
 
   }
   cy.log('After block COO completed');
+});
+
+// Apply incident fixture manifests to the cluster
+Cypress.Commands.add('createKubePodCrashLoopingAlert', () => {
+  const kubeconfigPath = Cypress.env('KUBECONFIG_PATH');
+  
+  // Generate a random alert name for this test run
+  const randomAlertName = `CustomPodCrashLooping_${Math.random().toString(36).substring(2, 15)}`;
+  
+  // Store the alert name globally so tests can access it
+  Cypress.env('CURRENT_ALERT_NAME', randomAlertName);
+  
+  cy.log(`Generated random alert name: ${randomAlertName}`);
+  
+  // Read the template and replace the placeholder
+  cy.readFile('./cypress/fixtures/incidents/prometheus_rule_pod_crash_loop.yaml').then((template) => {
+    const yamlContent = template.replace(/\{\{ALERT_NAME\}\}/g, randomAlertName);
+    
+    // Write the modified YAML to a temporary file
+    cy.writeFile('./cypress/fixtures/incidents/temp_prometheus_rule.yaml', yamlContent).then(() => {
+      // Apply the modified YAML
+      cy.exec(
+        `oc apply -f ./cypress/fixtures/incidents/temp_prometheus_rule.yaml --kubeconfig ${kubeconfigPath}`,
+      );
+      
+      // Clean up temporary file
+      cy.exec('rm ./cypress/fixtures/incidents/temp_prometheus_rule.yaml');
+    });
+  });
+  
+  cy.exec(
+    `oc apply -f ./cypress/fixtures/incidents/pod_crash_loop.yaml --kubeconfig ${kubeconfigPath}`,
+  );
+  
+  // Return the alert name for the test to use
+  return cy.wrap(randomAlertName);
+});
+
+// Clean up incident fixture manifests from the cluster
+Cypress.Commands.add('cleanupIncidentPrometheusRules', () => {
+  const kubeconfigPath = Cypress.env('KUBECONFIG_PATH');
+  
+  // Delete all PrometheusRules that match our pattern (kubernetes-monitoring-podcrash-rules)
+  // This ensures cleanup before tests and after tests
+  cy.exec(
+    `oc delete prometheusrule kubernetes-monitoring-podcrash-rules -n openshift-monitoring --kubeconfig ${kubeconfigPath} --ignore-not-found=true`,
+  );
+  
+  // Clear the environment variable if it exists
+  if (Cypress.env('CURRENT_ALERT_NAME')) {
+    Cypress.env('CURRENT_ALERT_NAME', null);
+  }
+  
+  cy.executeAndDelete(
+    `oc delete -f ./cypress/fixtures/incidents/pod_crash_loop.yaml --ignore-not-found=true --kubeconfig ${kubeconfigPath}`,
+  );
+});
+
+// Custom waitUntil with timeout message
+Cypress.Commands.add('waitUntilWithCustomTimeout', (
+  fn: () => any,
+  options: { interval: number; timeout: number; timeoutMessage: string }
+) => {
+  const { timeoutMessage, ...waitOptions } = options;
+
+  // Set up custom error handling before the waitUntil call
+  cy.on('fail', (err) => {
+    if (err.message.includes('Timed out retrying')) {
+      // Create a new error with the custom message
+      const customError = new Error(timeoutMessage);
+      customError.stack = err.stack;
+      throw customError;
+    }
+    // For any other errors, re-throw them unchanged
+    throw err;
+  });
+
+  // Execute the waitUntil with the original options (without timeoutMessage)
+  return cy.waitUntil(fn, waitOptions);
 });

web/cypress/support/index.ts

@@ -7,10 +7,17 @@ export const checkErrors = () =>
     assert.isTrue(!win.windowError, win.windowError);
   });
 
-  Cypress.on('uncaught:exception', (err, runnable) => {
-    // returning false here prevents Cypress from failing the test
-    // on a JavaScript exception
-    if (err.message.includes('ResizeObserver loop completed with undelivered notifications')) {
-      return false
-    }
-  });
+
+  // Ignore benign ResizeObserver errors globally so they don't fail tests
+// See: https://docs.cypress.io/api/cypress-api/catalog-of-events#Uncaught-Exceptions
+Cypress.on('uncaught:exception', (err) => {
+  const message = err?.message || String(err || '');
+  if (
+    message.includes('ResizeObserver loop limit exceeded') ||
+    message.includes('ResizeObserver loop completed with undelivered notifications') ||
+    message.includes('ResizeObserver')
+  ) {
+    return false;
+  }
+  // allow other errors to fail the test
+});

web/cypress/tsconfig.json

@@ -4,5 +4,8 @@
     "lib": ["es5", "dom"],
     "types": ["cypress", "node"]
   },
-  "include": ["**/*.ts"]
+
+  "include": [
+    "**/*.ts"
+  ]
 }