Refactored OTE and added new 3CMO OTE binary

Author: vr4manta

Dockerfile

@@ -2,13 +2,15 @@ FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25-openshift-4.22 AS
 WORKDIR /go/src/github.com/openshift/cluster-cloud-controller-manager-operator
 COPY . .
 RUN make build &&\
-    gzip /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cloud-controller-manager-aws-tests-ext
+    gzip /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/bin/cloud-controller-manager-aws-tests-ext &&\
+    gzip /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/bin/cluster-cloud-controller-manager-operator-tests-ext
 
 FROM registry.ci.openshift.org/ocp/4.22:base-rhel9
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cluster-controller-manager-operator .
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/config-sync-controllers .
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/azure-config-credentials-injector .
 COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/manifests manifests
-COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cloud-controller-manager-aws-tests-ext.gz /usr/bin/cloud-controller-manager-aws-tests-ext.gz
+COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/bin/cloud-controller-manager-aws-tests-ext.gz /usr/bin/cloud-controller-manager-aws-tests-ext.gz
+COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/bin/cluster-cloud-controller-manager-operator-tests-ext.gz /usr/bin/cluster-cloud-controller-manager-operator-tests-ext.gz
 
 LABEL io.openshift.release.operator true

Makefile

@@ -9,6 +9,7 @@ PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 CONTROLLER_GEN = go run ${PROJECT_DIR}/vendor/sigs.k8s.io/controller-tools/cmd/controller-gen
 ENVTEST = go run ${PROJECT_DIR}/vendor/sigs.k8s.io/controller-runtime/tools/setup-envtest
 GOLANGCI_LINT = go run ${PROJECT_DIR}/vendor/github.com/golangci/golangci-lint/v2/cmd/golangci-lint
+REPO_PATH   ?= github.com/openshift/cluster-cloud-controller-manager-operator
 
 HOME ?= /tmp/kubebuilder-testing
 ifeq ($(HOME), /)
@@ -37,7 +38,8 @@ unit:
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path --bin-dir $(PROJECT_DIR)/bin --index https://raw.githubusercontent.com/openshift/api/master/envtest-releases.yaml)" ./hack/ci-test.sh
 
 # Build operator binaries
-build: operator config-sync-controllers azure-config-credentials-injector cloud-controller-manager-aws-tests-ext
+.PHONY: build
+build: operator config-sync-controllers azure-config-credentials-injector cloud-controller-manager-aws-tests-ext cluster-cloud-controller-manager-operator-tests-ext
 
 operator:
 	go build -o bin/cluster-controller-manager-operator cmd/cluster-cloud-controller-manager-operator/main.go
@@ -49,12 +51,16 @@ azure-config-credentials-injector:
 	go build -o bin/azure-config-credentials-injector cmd/azure-config-credentials-injector/main.go
 
 cloud-controller-manager-aws-tests-ext:
-	cd cmd/cloud-controller-manager-aws-tests-ext && \
-	 GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOPROXY=$(GOPROXY) go build \
-		-trimpath \
-		-ldflags="$(LDFLAGS)" \
-		-o=../../bin/cloud-controller-manager-aws-tests-ext .
+	cd openshift-tests/aws-tests && \
+	mkdir -p ../bin && \
+	go build $(GOGCFLAGS) -o "../bin/cloud-controller-manager-aws-tests-ext" \
+	      -trimpath -ldflags "$(LD_FLAGS)" .
 
+cluster-cloud-controller-manager-operator-tests-ext:
+	cd openshift-tests/operator-tests && \
+	mkdir -p ../bin && \
+	go build $(GOGCFLAGS) -o "../bin/cluster-cloud-controller-manager-operator-tests-ext" \
+	      -trimpath -ldflags "$(LD_FLAGS)" .
 
 # Run against the configured Kubernetes cluster in ~/.kube/config
 run: verify manifests
@@ -82,9 +88,7 @@ lint:
 # Run go mod
 .PHONY: vendor
 vendor:
-	go mod tidy
-	go mod vendor
-	go mod verify
+	hack/go-mod.sh
 
 # Generate code
 generate:

cmd/cloud-controller-manager-aws-tests-ext/go.mod

@@ -0,0 +1,13 @@
+# Binaries
+/bin/
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool
+*.out

openshift-tests/.gitignore

@@ -0,0 +1,193 @@
+# OpenShift Cloud Controller Manager Operator Tests
+
+This directory contains OpenShift Tests Extension (OTE) binaries for testing the cluster-cloud-controller-manager-operator.
+
+## Structure
+
+The test suite is organized into separate sub-projects, each with independent dependencies:
+
+- **`aws-tests/`** - AWS-specific cloud controller manager tests
+  - `e2e/aws/` - AWS-specific test implementations
+  - `e2e/common/` - Common helper functions (feature gate checks)
+  - `main.go` - Binary entry point
+  - `go.mod` - Independent dependencies (includes AWS SDK)
+- **`operator-tests/`** - Multi-platform operator conformance tests
+  - `e2e/operator/` - Operator test implementations
+  - `e2e/common/` - Common helper functions (client config)
+  - `main.go` - Binary entry point
+  - `go.mod` - Independent dependencies (no AWS SDK)
+- **`bin/`** - Compiled test binaries
+
+## Test Binaries
+
+### 1. `cluster-cloud-controller-manager-operator-tests-ext`
+
+**Purpose:** General cloud controller manager operator tests that run on multiple platforms
+
+**Suites:**
+- `ccm/operator/conformance/parallel` - Parallel conformance tests
+- `ccm/operator/conformance/serial` - Serial conformance tests
+
+**Test Selection:**
+- Platform-agnostic tests that work across cloud providers
+- OpenShift-specific CCM operator functionality tests
+- Automatically filters tests based on platform labels (e.g., `[platform:vsphere]`)
+
+**Use Cases:**
+- General operator conformance testing
+- Multi-platform test runs
+- OpenShift-specific feature validation (e.g., VSphereMixedNodeEnv)
+
+### 2. `cloud-controller-manager-aws-tests-ext`
+
+**Purpose:** AWS-specific cloud controller manager tests
+
+**Suite:** `ccm/aws/conformance/parallel`
+
+**Test Selection:**
+- AWS load balancer tests (`[cloud-provider-aws-e2e] loadbalancer`)
+- AWS node tests (`[cloud-provider-aws-e2e] nodes`)
+- OpenShift AWS-specific tests (`[cloud-provider-aws-e2e-openshift]`)
+- Filters out ECR tests and Serial tests
+
+**Platform Filter:** AWS only (`PlatformEquals("aws")`)
+
+**Topology Exclusions:**
+- On SingleReplica topology, excludes:
+  - "nodes should label nodes with topology network info if instance is supported"
+  - "nodes should set zone-id topology label"
+
+
+## Test Organization
+
+### AWS Tests (`aws-tests/e2e/`)
+
+- `aws/helper.go` - AWS helper functions (feature gate checks, AWS clients)
+- `aws/loadbalancer.go` - AWS NLB and load balancer tests
+- `common/helper.go` - Feature gate checking (`IsFeatureEnabled`)
+
+### Operator Tests (`operator-tests/e2e/`)
+
+- `operator/vsphere_mixed_node.go` - VSphereMixedNodeEnv feature gate tests
+- `common/helper.go` - Client configuration (`NewClientConfigForTest`)
+
+### Test Prefixes
+
+- `[cloud-provider-aws-e2e-openshift]` - OpenShift-specific tests
+
+### Feature Gates Tested
+
+- `AWSServiceLBNetworkSecurityGroup` - Managed security groups for NLBs
+- `VSphereMixedNodeEnv` - Platform-type node labels on vSphere
+
+## Development
+
+### Adding New Tests
+
+1. Add test files to the appropriate sub-project's `e2e/` directory:
+   - `aws-tests/e2e/aws/` - AWS-specific tests
+   - `aws-tests/e2e/common/` - AWS-specific shared helpers
+   - `operator-tests/e2e/operator/` - Operator tests
+   - `operator-tests/e2e/common/` - Operator-specific shared helpers
+
+2. Import the test package in the binary's `main.go` (blank import for side effects):
+   ```go
+   import _ "github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/aws-tests/e2e/aws"
+   ```
+
+3. Tests are automatically discovered by the OTE framework via Ginkgo suite scanning
+
+**Note:** The `e2e/common/` directories are independent in each sub-project and contain only the helper functions needed by that specific binary.
+
+### Building Binaries
+
+```bash
+# From project root - build both binaries
+make cloud-controller-manager-aws-tests-ext cluster-cloud-controller-manager-operator-tests-ext
+
+# Or build all test binaries
+make build
+
+# Or build individually from within each sub-project
+cd openshift-tests/aws-tests && go build -o ../bin/cloud-controller-manager-aws-tests-ext .
+cd openshift-tests/operator-tests && go build -o ../bin/cluster-cloud-controller-manager-operator-tests-ext .
+```
+
+Binaries are built to `openshift-tests/bin/`:
+- `cloud-controller-manager-aws-tests-ext` (~95MB)
+- `cluster-cloud-controller-manager-operator-tests-ext` (~85MB)
+
+### Running Tests
+
+The test binaries are OpenShift Tests Extension (OTE) binaries and follow the OTE command structure:
+
+```bash
+# List available test suites
+./openshift-tests/bin/cloud-controller-manager-aws-tests-ext list
+
+# Run a specific suite
+./openshift-tests/bin/cloud-controller-manager-aws-tests-ext run ccm/aws/conformance/parallel
+
+# List operator tests
+./openshift-tests/bin/cluster-cloud-controller-manager-operator-tests-ext list
+
+# Run operator tests (parallel or serial)
+./openshift-tests/bin/cluster-cloud-controller-manager-operator-tests-ext run ccm/operator/conformance/parallel
+./openshift-tests/bin/cluster-cloud-controller-manager-operator-tests-ext run ccm/operator/conformance/serial
+```
+
+**Prerequisites:**
+- `KUBECONFIG` environment variable must be set
+- For AWS tests: AWS credentials configured (via `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` or instance profile)
+- For AWS tests: `AWS_REGION` or `LEASED_RESOURCE` environment variable set
+
+### Updating Dependencies
+
+Each sub-project has its own `go.mod` file, allowing independent dependency management:
+
+```bash
+# From project root - update all modules in the workspace
+./hack/go-mod.sh
+
+# Or update a specific sub-project independently
+cd openshift-tests/aws-tests && go mod tidy
+cd openshift-tests/operator-tests && go mod tidy
+```
+
+The `hack/go-mod.sh` script (run from project root):
+- Tidies all modules in the workspace (root, aws-tests, operator-tests)
+- Syncs workspace dependencies via `go work sync`
+- Re-tidies after sync to apply version bumps
+- Verifies all modules
+- Creates unified vendor directory
+
+**Benefits of separate modules:**
+- AWS SDK updates only affect `aws-tests/` - operator tests remain stable
+- Each binary can update dependencies independently without blocking the other
+- Faster, more focused dependency updates
+- Reduced dependency bloat (operator tests don't pull in AWS SDK)
+
+## Architecture
+
+### Go Workspace
+
+This directory contains multiple Go modules organized as a workspace:
+- Root module: `github.com/openshift/cluster-cloud-controller-manager-operator`
+- AWS tests: `github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/aws-tests`
+- Operator tests: `github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/operator-tests`
+
+All modules are coordinated via the workspace defined in `../go.work`.
+
+### OTE Framework
+
+Uses OpenShift Tests Extension framework:
+- Test discovery via Ginkgo suite scanning
+- Extension registration and suite management
+- Platform and topology filtering
+- Kubernetes E2E framework integration
+
+## References
+
+- [OpenShift Tests Extension](https://github.com/openshift-eng/openshift-tests-extension)
+- [Kubernetes E2E Framework](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/framework)
+- [Cloud Provider AWS Tests](https://github.com/kubernetes/cloud-provider-aws/tree/master/tests/e2e)

openshift-tests/README.md

@@ -1,4 +1,4 @@
-package e2e
+package aws
 
 import (
 	"context"
@@ -11,8 +11,7 @@ import (
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
 	elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
-	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/aws-tests/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -60,64 +59,10 @@ func getAWSLoadBalancerFromDNSName(ctx context.Context, elbClient *elbv2.Client,
 	return foundLB, nil
 }
 
-// isFeatureEnabled checks if an OpenShift feature gate is enabled by querying the
-// FeatureGate resource named "cluster" using the typed OpenShift config API.
-//
-// This function uses the official OpenShift config/v1 API types for type-safe
-// access to feature gate information, providing better performance and maintainability
-// compared to dynamic client approaches.
-//
-// Parameters:
-//   - ctx: Context for the API call
-//   - featureName: Name of the feature gate to check (e.g., "AWSServiceLBNetworkSecurityGroup")
-//
-// Returns:
-//   - bool: true if the feature is enabled, false if disabled or not found
-//   - error: error if the API call fails
-//
-// Note: For HyperShift clusters, this checks the management cluster's feature gates.
-// To check hosted cluster feature gates, use the hosted cluster's kubeconfig.
+// isFeatureEnabled is a convenience wrapper around common.IsFeatureEnabled.
+// Deprecated: Use common.IsFeatureEnabled directly instead.
 func isFeatureEnabled(ctx context.Context, featureName string) (bool, error) {
-	// Get the REST config
-	restConfig, err := framework.LoadConfig()
-	if err != nil {
-		return false, fmt.Errorf("failed to load kubeconfig: %v", err)
-	}
-
-	// Create typed config client (more efficient than dynamic client)
-	configClient, err := configv1client.NewForConfig(restConfig)
-	if err != nil {
-		return false, fmt.Errorf("failed to create config client: %v", err)
-	}
-
-	// Get the FeatureGate resource using typed API
-	featureGate, err := configClient.FeatureGates().Get(ctx, "cluster", metav1.GetOptions{})
-	if err != nil {
-		return false, fmt.Errorf("failed to get FeatureGate 'cluster': %v", err)
-	}
-
-	// Iterate through the feature gates status (typed structs)
-	for _, fg := range featureGate.Status.FeatureGates {
-		// Check enabled list
-		for _, enabled := range fg.Enabled {
-			if string(enabled.Name) == featureName {
-				framework.Logf("Feature %s is enabled (version %s)", featureName, fg.Version)
-				return true, nil
-			}
-		}
-
-		// Check disabled list
-		for _, disabled := range fg.Disabled {
-			if string(disabled.Name) == featureName {
-				framework.Logf("Feature %s is disabled (version %s)", featureName, fg.Version)
-				return false, nil
-			}
-		}
-	}
-
-	// Feature not found in either list
-	framework.Logf("Feature %s not found in FeatureGate status", featureName)
-	return false, nil
+	return common.IsFeatureEnabled(ctx, featureName)
 }
 
 // getAWSClientEC2 creates an AWS EC2 client using default credentials configured in the environment.

…ller-manager-aws-tests-ext/e2e/helper.go …nshift-tests/aws-tests/e2e/aws/helper.gocmd/cloud-controller-manager-aws-tests-ext/e2e/helper.go renamed to openshift-tests/aws-tests/e2e/aws/helper.go cmd/cloud-controller-manager-aws-tests-ext/e2e/helper.go renamed to openshift-tests/aws-tests/e2e/aws/helper.go

@@ -1,4 +1,4 @@
-package e2e
+package aws
 
 import (
 	"context"