MGMT-20128: Image based install operator not reconciling after pod is restarted

The monitor will list existing ImageCLusterInstall CRs and call client.Status.Update for all ICI that didn't complete the installation
This should enqueu the ICI CRs for reconcilation

Signed-off-by: Eran Cohen <eranco@redhat.com>

Author: eranco74

cmd/manager/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"net/http"
@@ -28,11 +29,8 @@ import (
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 
-	"github.com/kelseyhightower/envconfig"
-	bmh_v1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
-	hivev1 "github.com/openshift/hive/apis/hive/v1"
-	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -46,6 +44,11 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
+	"github.com/kelseyhightower/envconfig"
+	bmh_v1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+	hivev1 "github.com/openshift/hive/apis/hive/v1"
+	"github.com/sirupsen/logrus"
+
 	"github.com/openshift/image-based-install-operator/api/v1alpha1"
 	"github.com/openshift/image-based-install-operator/controllers"
 	"github.com/openshift/image-based-install-operator/internal/credentials"
@@ -191,13 +194,44 @@ func main() {
 		os.Exit(1)
 	}
 
+	go EnqueueExistingImageClusterInstall(mgr)
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}
 }
 
+func EnqueueExistingImageClusterInstall(mgr manager.Manager) {
+	ctx := context.Background()
+
+	// Wait for cache to sync
+	setupLog.Info("Waiting for cache to sync...")
+	if synced := mgr.GetCache().WaitForCacheSync(ctx); !synced {
+		setupLog.Error(fmt.Errorf("Failed to wait for cache to sync"), "failed to wait for cache to sync while enqueuing existing ImageClusterInstall")
+		os.Exit(1)
+	}
+	setupLog.Info("Cache successfully started and synced")
+
+	// List existing resources and enqueue those with ClusterInstallCompleted condition false
+	var icilList v1alpha1.ImageClusterInstallList
+	if err := mgr.GetClient().List(ctx, &icilList); err != nil {
+		setupLog.Error(err, "Failed to list ImageClusterInstall")
+		os.Exit(1)
+	}
+
+	for _, ici := range icilList.Items {
+		if !controllers.InstallationCompleted(&ici) {
+			setupLog.Info("Enqueuing existing ImageClusterInstall:", "namespace", ici.Namespace, "name", ici.Name)
+			// Triggers reconciliation
+			if err := mgr.GetClient().Status().Update(ctx, &ici); err != nil {
+				setupLog.Info("Failed to requeue ImageClusterInstall", "namespace", ici.Namespace, "name", ici.Name)
+			}
+		}
+	}
+}
+
 func serviceURL(opts *controllers.ImageClusterInstallReconcilerOptions) (string, error) {
 	if opts.ServiceName == "" || opts.ServiceNamespace == "" || opts.ServiceScheme == "" {
 		return "", fmt.Errorf("SERVICE_NAME, SERVICE_NAMESPACE, and SERVICE_SCHEME must be set")

controllers/conditions.go

@@ -165,7 +165,7 @@ func installationTimedout(ici *v1alpha1.ImageClusterInstall) bool {
 	return cond != nil && cond.Status == corev1.ConditionTrue && cond.Reason == v1alpha1.InstallTimedoutReason
 }
 
-func installationCompleted(ici *v1alpha1.ImageClusterInstall) bool {
+func InstallationCompleted(ici *v1alpha1.ImageClusterInstall) bool {
 	cond := findCondition(ici.Status.Conditions, hivev1.ClusterInstallCompleted)
 	return cond != nil && cond.Status == corev1.ConditionTrue
 }

controllers/imageclusterinstall_controller.go

@@ -143,7 +143,7 @@ func (r *ImageClusterInstallReconciler) Reconcile(ctx context.Context, req ctrl.
 	}
 
 	// Nothing to do if the installation is complete
-	if installationCompleted(ici) {
+	if InstallationCompleted(ici) {
 		return ctrl.Result{}, nil
 	}
 	// Nothing to do if the installation process started and the config.iso exists

controllers/imageclusterinstall_monitor.go

@@ -18,7 +18,6 @@ package controllers
 
 import (
 	"context"
-
 	// These are required for image parsing to work correctly with digest-based pull specs
 	// See: https://github.com/opencontainers/go-digest/blob/v1.0.0/README.md#usage
 	_ "crypto/sha256"
@@ -74,7 +73,7 @@ func (r *ImageClusterInstallMonitor) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{}, nil
 	}
 	// Nothing to do if the installation process has already stopped
-	if installationCompleted(ici) {
+	if InstallationCompleted(ici) {
 		log.Infof("Cluster %s/%s finished installation process, nothing to do", ici.Namespace, ici.Name)
 		return ctrl.Result{}, nil
 	}