[WIP] ⚠️ Allow specifying a different credentials per VSphereMachine

apis/v1beta1/types.go

@@ -209,6 +209,10 @@ type VirtualMachineCloneSpec struct {
 	// +listMapKey=name
 	// +kubebuilder:validation:MaxItems=29
 	DataDisks []VSphereDisk `json:"dataDisks,omitempty"`
+	// IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains
+	// the identity to use when reconciling the virtual machine.
+	// +optional
+	IdentityRef *VSphereIdentityReference `json:"identityRef,omitempty"`
 }
 
 // VSphereDisk is an additional disk to add to the VM that is not part of the VM OVA template.

config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml

@@ -1053,6 +1053,26 @@ spec:
                   virtual machine is cloned.
                   Check the compatibility with the ESXi version before setting the value.
                 type: string
+              identityRef:
+                description: |-
+                  IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains
+                  the identity to use when reconciling the virtual machine.
+                properties:
+                  kind:
+                    description: Kind of the identity. Can either be VSphereClusterIdentity
+                      or Secret
+                    enum:
+                    - VSphereClusterIdentity
+                    - Secret
+                    type: string
+                  name:
+                    description: Name of the identity.
+                    minLength: 1
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
               memoryMiB:
                 description: |-
                   MemoryMiB is the size of a virtual machine's memory, in MiB.

config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml

@@ -923,6 +923,26 @@ spec:
                           virtual machine is cloned.
                           Check the compatibility with the ESXi version before setting the value.
                         type: string
+                      identityRef:
+                        description: |-
+                          IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains
+                          the identity to use when reconciling the virtual machine.
+                        properties:
+                          kind:
+                            description: Kind of the identity. Can either be VSphereClusterIdentity
+                              or Secret
+                            enum:
+                            - VSphereClusterIdentity
+                            - Secret
+                            type: string
+                          name:
+                            description: Name of the identity.
+                            minLength: 1
+                            type: string
+                        required:
+                        - kind
+                        - name
+                        type: object
                       memoryMiB:
                         description: |-
                           MemoryMiB is the size of a virtual machine's memory, in MiB.

config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml

@@ -1138,6 +1138,26 @@ spec:
                   virtual machine is cloned.
                   Check the compatibility with the ESXi version before setting the value.
                 type: string
+              identityRef:
+                description: |-
+                  IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains
+                  the identity to use when reconciling the virtual machine.
+                properties:
+                  kind:
+                    description: Kind of the identity. Can either be VSphereClusterIdentity
+                      or Secret
+                    enum:
+                    - VSphereClusterIdentity
+                    - Secret
+                    type: string
+                  name:
+                    description: Name of the identity.
+                    minLength: 1
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
               memoryMiB:
                 description: |-
                   MemoryMiB is the size of a virtual machine's memory, in MiB.

controllers/vspherevm_controller.go

@@ -588,14 +588,25 @@ func (r vmReconciler) ipAddressClaimToVSphereVM(_ context.Context, a ctrlclient.
 
 func (r vmReconciler) retrieveVcenterSession(ctx context.Context, vsphereVM *infrav1.VSphereVM) (*session.Session, error) {
 	log := ctrl.LoggerFrom(ctx)
-	// Get cluster object and then get VSphereCluster object
 
 	params := session.NewParams().
 		WithServer(vsphereVM.Spec.Server).
 		WithDatacenter(vsphereVM.Spec.Datacenter).
-		WithUserInfo(r.ControllerManagerContext.Username, r.ControllerManagerContext.Password).
 		WithThumbprint(vsphereVM.Spec.Thumbprint)
 
+	// if there is an identityref coming with the vsphereVM, we use that regardless of the state/existence of the cluster & vspherecluster
+	if vsphereVM.Spec.IdentityRef != nil {
+		creds, err := identity.GetCredentialsFromVshpereVM(ctx, r.Client, vsphereVM, r.Namespace)
+		if err != nil {
+			return nil, err
+		}
+		params = params.WithUserInfo(creds.Username, creds.Password)
+		log.V(4).Info("Using credentials attached to the VsphereVM")
+		return session.GetOrCreate(ctx, params)
+	} else {
+		// if the vsphereVM doesn't have an identityRef, set the default user identity to that provided by the ControllerManager
+		params = params.WithUserInfo(r.ControllerManagerContext.Username, r.ControllerManagerContext.Password)
+	}
 	cluster, err := clusterutilv1.GetClusterFromMetadata(ctx, r.Client, vsphereVM.ObjectMeta)
 	if err != nil {
 		log.V(4).Info("Using credentials provided to the manager to create the authenticated session, VSphereVM is missing cluster label or cluster does not exist")

docs/multi-vcenter.md

@@ -0,0 +1,39 @@
+# Multi VCenter support
+
+Cluster API Provider vSphere (CAPV) supports multiple VCenter for a single. Therefore CAPV is allowing to define the used identity for each machine. CAPV will check on every Machine first, if there is a local identity otherwise it fallback on the default selection method. 
+
+In order to run a CAPV cluster in multiple VCenter, you have to configure CPI & CSI to support multi VCenter, see [guide](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-8B3B9004-DE37-4E6B-9AA1-234CDA1BD7F9.html). Trivia, `VSphereCluster` can be only in single VCenter. This will just used as a fallback, if you haven't configured a different identity for a `VSphereMachine``.
+
+## Examples
+
+Deploy a `VSphereMachine` with a custom identityRef:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachine
+metadata:
+  name: new-workload-cluster
+spec:
+  server: vcenter
+  identityRef:
+    kind: VSphereClusterIdentity
+    name: identityName
+...
+```
+
+Deploy a `VSphereMachineTemplate` with a custom identityRef:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: new-workload-cluster
+spec:
+  template:
+    spec:
+       server: vcenter
+      identityRef:
+        kind: VSphereClusterIdentity
+        name: identityName
+...
+```

pkg/identity/identity.go

@@ -44,9 +44,76 @@ type Credentials struct {
 	Password string
 }
 
-// GetCredentials returns the VCenter credentials for the VSphereCluster.
+func GetCredentialsFromVshpereVM(ctx context.Context, c client.Client, machine *infrav1.VSphereVM, controllerNamespace string) (*Credentials, error) {
+	if err := validateInputs(c, machine.Namespace, machine.Spec.IdentityRef); err != nil {
+		return nil, err
+	}
+
+	ref := machine.Spec.IdentityRef
+	secret := &corev1.Secret{}
+	var secretKey client.ObjectKey
+
+	switch ref.Kind {
+	case infrav1.SecretKind:
+		secretKey = client.ObjectKey{
+			Namespace: machine.Namespace,
+			Name:      ref.Name,
+		}
+	case infrav1.VSphereClusterIdentityKind:
+		identity := &infrav1.VSphereClusterIdentity{}
+		key := client.ObjectKey{
+			Name: ref.Name,
+		}
+		if err := c.Get(ctx, key, identity); err != nil {
+			return nil, err
+		}
+
+		if !identity.Status.Ready {
+			return nil, errors.New("identity isn't ready to be used yet")
+		}
+
+		if identity.Spec.AllowedNamespaces == nil {
+			return nil, errors.New("allowedNamespaces set to nil, no namespaces are allowed to use this identity")
+		}
+
+		selector, err := metav1.LabelSelectorAsSelector(&identity.Spec.AllowedNamespaces.Selector)
+		if err != nil {
+			return nil, errors.New("failed to build selector")
+		}
+
+		ns := &corev1.Namespace{}
+		nsKey := client.ObjectKey{
+			Name: machine.Namespace,
+		}
+		if err := c.Get(ctx, nsKey, ns); err != nil {
+			return nil, err
+		}
+		if !selector.Matches(labels.Set(ns.GetLabels())) {
+			return nil, fmt.Errorf("namespace %s is not allowed to use specifified identity", machine.Namespace)
+		}
+
+		secretKey = client.ObjectKey{
+			Name:      identity.Spec.SecretName,
+			Namespace: controllerNamespace,
+		}
+	default:
+		return nil, fmt.Errorf("unknown type %s used for Identity", ref.Kind)
+	}
+
+	if err := c.Get(ctx, secretKey, secret); err != nil {
+		return nil, err
+	}
+
+	credentials := &Credentials{
+		Username: getData(secret, UsernameKey),
+		Password: getData(secret, PasswordKey),
+	}
+
+	return credentials, nil
+}
+
 func GetCredentials(ctx context.Context, c client.Client, cluster *infrav1.VSphereCluster, controllerNamespace string) (*Credentials, error) {
-	if err := validateInputs(c, cluster); err != nil {
+	if err := validateInputs(c, cluster.Namespace, cluster.Spec.IdentityRef); err != nil {
 		return nil, err
 	}
 
@@ -113,15 +180,14 @@ func GetCredentials(ctx context.Context, c client.Client, cluster *infrav1.VSphe
 	return credentials, nil
 }
 
-func validateInputs(c client.Client, cluster *infrav1.VSphereCluster) error {
+func validateInputs(c client.Client, namespace string, identityRef *infrav1.VSphereIdentityReference) error {
 	if c == nil {
 		return errors.New("kubernetes client is required")
 	}
-	if cluster == nil {
+	if namespace == "" {
 		return errors.New("vsphere cluster is required")
 	}
-	ref := cluster.Spec.IdentityRef
-	if ref == nil {
+	if identityRef == nil {
 		return errors.New("IdentityRef is required")
 	}
 	return nil

pkg/identity/identity_test.go

@@ -235,19 +235,19 @@ var _ = Describe("validateInputs", func() {
 
 	Context("If the client is missing", func() {
 		It("should error if client is missing", func() {
-			Expect(validateInputs(nil, cluster)).NotTo(Succeed())
+			Expect(validateInputs(nil, cluster.Namespace, nil)).NotTo(Succeed())
 		})
 	})
 
 	Context("If the cluster is missing", func() {
 		It("should error if cluster is missing", func() {
-			Expect(validateInputs(k8sclient, nil)).NotTo(Succeed())
+			Expect(validateInputs(k8sclient, "", nil)).NotTo(Succeed())
 		})
 	})
 
 	Context("If the identityRef is missing on cluster", func() {
 		It("should error if identityRef is missing on cluster", func() {
-			Expect(validateInputs(k8sclient, cluster)).NotTo(Succeed())
+			Expect(validateInputs(k8sclient, cluster.Namespace, nil)).NotTo(Succeed())
 		})
 	})
 })