Merge pull request #18197 from hakman/azure-ccm

azure: Deploy cloud-controller-manager for node lifecycle and LB support

Author: k8s-ci-robot

.gitignore

@@ -92,3 +92,6 @@ _output
 # Used by E2E testing
 _artifacts
 _rundir
+
+# kustomize helm chart cache
+charts/

k8s/crds/kops.k8s.io_clusters.yaml

@@ -666,6 +666,10 @@ spec:
                     description: Allow the cluster to run without the cluster-id on
                       cloud instances
                     type: boolean
+                  azureNodeManagerImage:
+                    description: AzureNodeManagerImage is the OCI image of the Azure
+                      cloud node manager.
+                    type: string
                   cidrAllocatorType:
                     description: CIDRAllocatorType specifies the type of CIDR allocator
                       to use.

nodeup/pkg/model/cloudconfig.go

@@ -37,33 +37,22 @@ const (
 )
 
 // azureCloudConfig is the configuration passed to Cloud Provider Azure.
-// The specification is described in https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/.
+// The specification is described in https://cloud-provider-azure.sigs.k8s.io/install/configs/.
+// Field order follows the upstream documentation: auth configs first, then cluster config.
 type azureCloudConfig struct {
-	// SubscriptionID is the ID of the Azure Subscription that the cluster is deployed in.
-	SubscriptionID string `json:"subscriptionId,omitempty"`
-	// TenantID is the ID of the tenant that the cluster is deployed in.
-	TenantID string `json:"tenantId"`
-	// CloudConfigType is the cloud configure type for Azure cloud provider. Supported values are file, secret and merge.
-	CloudConfigType string `json:"cloudConfigType,omitempty"`
-	// VMType is the type of azure nodes.
-	VMType string `json:"vmType,omitempty" yaml:"vmType,omitempty"`
-	// ResourceGroup is the name of the resource group that the cluster is deployed in.
-	ResourceGroup string `json:"resourceGroup,omitempty"`
-	// Location is the location of the resource group that the cluster is deployed in.
-	Location string `json:"location,omitempty"`
-	// RouteTableName is the name of the route table attached to the subnet that the cluster is deployed in.
-	RouteTableName string `json:"routeTableName,omitempty"`
-	// VnetName is the name of the virtual network that the cluster is deployed in.
-	VnetName string `json:"vnetName"`
-
-	// UseInstanceMetadata specifies where instance metadata service is used where possible.
-	UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty"`
-	// UseManagedIdentityExtension specifies where managed service
-	// identity is used for the virtual machine to access Azure
-	// ARM APIs.
-	UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty"`
-	// DisableAvailabilitySetNodes disables VMAS nodes support.
-	DisableAvailabilitySetNodes bool `json:"disableAvailabilitySetNodes,omitempty"`
+	// Auth configs
+	TenantID                    string `json:"tenantId,omitempty"`
+	SubscriptionID              string `json:"subscriptionId,omitempty"`
+	UseManagedIdentityExtension bool   `json:"useManagedIdentityExtension,omitempty"`
+
+	// Cluster config
+	ResourceGroup               string `json:"resourceGroup,omitempty"`
+	Location                    string `json:"location,omitempty"`
+	VnetName                    string `json:"vnetName,omitempty"`
+	SubnetName                  string `json:"subnetName,omitempty"`
+	SecurityGroupName           string `json:"securityGroupName,omitempty"`
+	UseInstanceMetadata         bool   `json:"useInstanceMetadata,omitempty"`
+	DisableAvailabilitySetNodes bool   `json:"disableAvailabilitySetNodes,omitempty"`
 }
 
 // CloudConfigBuilder creates the cloud configuration file
@@ -130,17 +119,17 @@ func (b *CloudConfigBuilder) build(c *fi.NodeupModelBuilderContext, inTree bool)
 		}
 
 		c := &azureCloudConfig{
-			CloudConfigType:             "file",
-			SubscriptionID:              b.NodeupConfig.AzureSubscriptionID,
+			// Auth
 			TenantID:                    b.NodeupConfig.AzureTenantID,
-			Location:                    b.NodeupConfig.AzureLocation,
-			VMType:                      "vmss",
+			SubscriptionID:              b.NodeupConfig.AzureSubscriptionID,
+			UseManagedIdentityExtension: true,
+			// Cluster
 			ResourceGroup:               b.NodeupConfig.AzureResourceGroup,
-			RouteTableName:              b.NodeupConfig.AzureRouteTableName,
+			Location:                    b.NodeupConfig.AzureLocation,
 			VnetName:                    vnetName,
+			SubnetName:                  b.NodeupConfig.AzureSubnetName,
+			SecurityGroupName:           b.NodeupConfig.AzureSecurityGroupName,
 			UseInstanceMetadata:         true,
-			UseManagedIdentityExtension: true,
-			// Disable availability set nodes as we currently use VMSS.
 			DisableAvailabilitySetNodes: true,
 		}
 		data, err := json.Marshal(c)

nodeup/pkg/model/cloudconfig_test.go

@@ -35,7 +35,6 @@ func TestBuildAzure(t *testing.T) {
 		subscriptionID    = "subID"
 		tenantID          = "tenantID"
 		resourceGroupName = "test-resource-group"
-		routeTableName    = "test-route-table"
 		vnetName          = "test-vnet"
 	)
 	cluster := &kops.Cluster{
@@ -48,7 +47,6 @@ func TestBuildAzure(t *testing.T) {
 					SubscriptionID:    subscriptionID,
 					TenantID:          tenantID,
 					ResourceGroupName: resourceGroupName,
-					RouteTableName:    routeTableName,
 				},
 			},
 			Networking: kops.NetworkingSpec{
@@ -101,14 +99,13 @@ func TestBuildAzure(t *testing.T) {
 		t.Fatalf("unexpected error from json.Unmarshal(%q): %v", string(data), err)
 	}
 	expected := azureCloudConfig{
-		CloudConfigType:             "file",
 		SubscriptionID:              subscriptionID,
 		TenantID:                    tenantID,
 		Location:                    "eastus",
-		VMType:                      "vmss",
 		ResourceGroup:               resourceGroupName,
-		RouteTableName:              routeTableName,
 		VnetName:                    vnetName,
+		SubnetName:                  "test-subnet",
+		SecurityGroupName:           vnetName,
 		UseInstanceMetadata:         true,
 		UseManagedIdentityExtension: true,
 		DisableAvailabilitySetNodes: true,

pkg/apis/kops/cluster.go

@@ -922,6 +922,15 @@ func (c *Cluster) AzureRouteTableName() string {
 	return c.Name
 }
 
+// AzureNetworkSecurityGroupName returns the name of the network security group for the cluster.
+// The NSG shares its name with the virtual network.
+func (c *Cluster) AzureNetworkSecurityGroupName() string {
+	if c.Spec.Networking.NetworkID != "" {
+		return c.Spec.Networking.NetworkID
+	}
+	return c.Name
+}
+
 func (c *Cluster) PublishesDNSRecords() bool {
 	if c.UsesNoneDNS() || dns.IsGossipClusterName(c.Name) {
 		return false

pkg/apis/kops/componentconfig.go

@@ -779,6 +779,8 @@ type CloudControllerManagerConfig struct {
 	NodeStatusUpdateFrequency *metav1.Duration `json:"nodeStatusUpdateFrequency,omitempty" flag:"node-status-update-frequency"`
 	// ConcurrentNodeSyncs is the number of workers concurrently synchronizing nodes. (default: 1)
 	ConcurrentNodeSyncs *int32 `json:"concurrentNodeSyncs,omitempty" flag:"concurrent-node-syncs"`
+	// AzureNodeManagerImage is the OCI image of the Azure cloud node manager.
+	AzureNodeManagerImage string `json:"azureNodeManagerImage,omitempty"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -785,6 +785,8 @@ type CloudControllerManagerConfig struct {
 	NodeStatusUpdateFrequency *metav1.Duration `json:"nodeStatusUpdateFrequency,omitempty" flag:"node-status-update-frequency"`
 	// ConcurrentNodeSyncs is the number of workers concurrently synchronizing nodes. (default: 1)
 	ConcurrentNodeSyncs *int32 `json:"concurrentNodeSyncs,omitempty" flag:"concurrent-node-syncs"`
+	// AzureNodeManagerImage is the OCI image of the Azure cloud node manager.
+	AzureNodeManagerImage string `json:"azureNodeManagerImage,omitempty"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -776,6 +776,8 @@ type CloudControllerManagerConfig struct {
 	NodeStatusUpdateFrequency *metav1.Duration `json:"nodeStatusUpdateFrequency,omitempty" flag:"node-status-update-frequency"`
 	// ConcurrentNodeSyncs is the number of workers concurrently synchronizing nodes. (default: 1)
 	ConcurrentNodeSyncs *int32 `json:"concurrentNodeSyncs,omitempty" flag:"concurrent-node-syncs"`
+	// AzureNodeManagerImage is the OCI image of the Azure cloud node manager.
+	AzureNodeManagerImage string `json:"azureNodeManagerImage,omitempty"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler