From 70819fe0d690b05c7f116052f35d4230d0c63045 Mon Sep 17 00:00:00 2001 From: Gabriel Quennesson Date: Sun, 9 Feb 2025 21:53:46 +0100 Subject: [PATCH] :sparkles: Add support for defining an identity in the vspheremachine and defaulting to this indentity if it exists. --- apis/v1alpha3/zz_generated.conversion.go | 1 + apis/v1alpha4/zz_generated.conversion.go | 1 + apis/v1beta1/types.go | 4 + apis/v1beta1/zz_generated.deepcopy.go | 5 ++ ...ture.cluster.x-k8s.io_vspheremachines.yaml | 20 +++++ ...ster.x-k8s.io_vspheremachinetemplates.yaml | 20 +++++ ...structure.cluster.x-k8s.io_vspherevms.yaml | 20 +++++ controllers/vspherevm_controller.go | 15 +++- docs/multi-vcenter.md | 39 ++++++++++ pkg/identity/identity.go | 78 +++++++++++++++++-- pkg/identity/identity_test.go | 6 +- 11 files changed, 198 insertions(+), 11 deletions(-) create mode 100644 docs/multi-vcenter.md diff --git a/apis/v1alpha3/zz_generated.conversion.go b/apis/v1alpha3/zz_generated.conversion.go index f92ba0b018..30aa9bd6fe 100644 --- a/apis/v1alpha3/zz_generated.conversion.go +++ b/apis/v1alpha3/zz_generated.conversion.go @@ -1781,5 +1781,6 @@ func autoConvert_v1beta1_VirtualMachineCloneSpec_To_v1alpha3_VirtualMachineClone // WARNING: in.OS requires manual conversion: does not exist in peer-type // WARNING: in.HardwareVersion requires manual conversion: does not exist in peer-type // WARNING: in.DataDisks requires manual conversion: does not exist in peer-type + // WARNING: in.IdentityRef requires manual conversion: does not exist in peer-type return nil } diff --git a/apis/v1alpha4/zz_generated.conversion.go b/apis/v1alpha4/zz_generated.conversion.go index 55e25811bc..9932af0f27 100644 --- a/apis/v1alpha4/zz_generated.conversion.go +++ b/apis/v1alpha4/zz_generated.conversion.go @@ -1935,5 +1935,6 @@ func autoConvert_v1beta1_VirtualMachineCloneSpec_To_v1alpha4_VirtualMachineClone // WARNING: in.OS requires manual conversion: does not exist in peer-type // WARNING: in.HardwareVersion requires manual conversion: does not exist in peer-type // WARNING: in.DataDisks requires manual conversion: does not exist in peer-type + // WARNING: in.IdentityRef requires manual conversion: does not exist in peer-type return nil } diff --git a/apis/v1beta1/types.go b/apis/v1beta1/types.go index 4652d201dd..c4ebd8b465 100644 --- a/apis/v1beta1/types.go +++ b/apis/v1beta1/types.go @@ -209,6 +209,10 @@ type VirtualMachineCloneSpec struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=29 DataDisks []VSphereDisk `json:"dataDisks,omitempty"` + // IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains + // the identity to use when reconciling the virtual machine. + // +optional + IdentityRef *VSphereIdentityReference `json:"identityRef,omitempty"` } // VSphereDisk is an additional disk to add to the VM that is not part of the VM OVA template. diff --git a/apis/v1beta1/zz_generated.deepcopy.go b/apis/v1beta1/zz_generated.deepcopy.go index 3d338d028f..4926366139 100644 --- a/apis/v1beta1/zz_generated.deepcopy.go +++ b/apis/v1beta1/zz_generated.deepcopy.go @@ -1425,6 +1425,11 @@ func (in *VirtualMachineCloneSpec) DeepCopyInto(out *VirtualMachineCloneSpec) { *out = make([]VSphereDisk, len(*in)) copy(*out, *in) } + if in.IdentityRef != nil { + in, out := &in.IdentityRef, &out.IdentityRef + *out = new(VSphereIdentityReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineCloneSpec. diff --git a/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml b/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml index 5387b31ecd..819b7259e7 100644 --- a/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml +++ b/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml @@ -1053,6 +1053,26 @@ spec: virtual machine is cloned. Check the compatibility with the ESXi version before setting the value. type: string + identityRef: + description: |- + IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains + the identity to use when reconciling the virtual machine. + properties: + kind: + description: Kind of the identity. Can either be VSphereClusterIdentity + or Secret + enum: + - VSphereClusterIdentity + - Secret + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object memoryMiB: description: |- MemoryMiB is the size of a virtual machine's memory, in MiB. diff --git a/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml b/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml index f7fa965da5..bc620df348 100644 --- a/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml +++ b/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml @@ -923,6 +923,26 @@ spec: virtual machine is cloned. Check the compatibility with the ESXi version before setting the value. type: string + identityRef: + description: |- + IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains + the identity to use when reconciling the virtual machine. + properties: + kind: + description: Kind of the identity. Can either be VSphereClusterIdentity + or Secret + enum: + - VSphereClusterIdentity + - Secret + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object memoryMiB: description: |- MemoryMiB is the size of a virtual machine's memory, in MiB. diff --git a/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml b/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml index 01bad7a087..51fcb6ad61 100644 --- a/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml +++ b/config/default/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml @@ -1138,6 +1138,26 @@ spec: virtual machine is cloned. Check the compatibility with the ESXi version before setting the value. type: string + identityRef: + description: |- + IdentityRef is a reference to either a Secret or VSphereClusterIdentity that contains + the identity to use when reconciling the virtual machine. + properties: + kind: + description: Kind of the identity. Can either be VSphereClusterIdentity + or Secret + enum: + - VSphereClusterIdentity + - Secret + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object memoryMiB: description: |- MemoryMiB is the size of a virtual machine's memory, in MiB. diff --git a/controllers/vspherevm_controller.go b/controllers/vspherevm_controller.go index 5a5dd0a89b..e7985e0332 100644 --- a/controllers/vspherevm_controller.go +++ b/controllers/vspherevm_controller.go @@ -588,14 +588,25 @@ func (r vmReconciler) ipAddressClaimToVSphereVM(_ context.Context, a ctrlclient. func (r vmReconciler) retrieveVcenterSession(ctx context.Context, vsphereVM *infrav1.VSphereVM) (*session.Session, error) { log := ctrl.LoggerFrom(ctx) - // Get cluster object and then get VSphereCluster object params := session.NewParams(). WithServer(vsphereVM.Spec.Server). WithDatacenter(vsphereVM.Spec.Datacenter). - WithUserInfo(r.ControllerManagerContext.Username, r.ControllerManagerContext.Password). WithThumbprint(vsphereVM.Spec.Thumbprint) + // if there is an identityref coming with the vsphereVM, we use that regardless of the state/existence of the cluster & vspherecluster + if vsphereVM.Spec.IdentityRef != nil { + creds, err := identity.GetCredentialsFromVshpereVM(ctx, r.Client, vsphereVM, r.Namespace) + if err != nil { + return nil, err + } + params = params.WithUserInfo(creds.Username, creds.Password) + log.V(4).Info("Using credentials attached to the VsphereVM") + return session.GetOrCreate(ctx, params) + } else { + // if the vsphereVM doesn't have an identityRef, set the default user identity to that provided by the ControllerManager + params = params.WithUserInfo(r.ControllerManagerContext.Username, r.ControllerManagerContext.Password) + } cluster, err := clusterutilv1.GetClusterFromMetadata(ctx, r.Client, vsphereVM.ObjectMeta) if err != nil { log.V(4).Info("Using credentials provided to the manager to create the authenticated session, VSphereVM is missing cluster label or cluster does not exist") diff --git a/docs/multi-vcenter.md b/docs/multi-vcenter.md new file mode 100644 index 0000000000..8aa28ffc65 --- /dev/null +++ b/docs/multi-vcenter.md @@ -0,0 +1,39 @@ +# Multi VCenter support + +Cluster API Provider vSphere (CAPV) supports multiple VCenter for a single. Therefore CAPV is allowing to define the used identity for each machine. CAPV will check on every Machine first, if there is a local identity otherwise it fallback on the default selection method. + +In order to run a CAPV cluster in multiple VCenter, you have to configure CPI & CSI to support multi VCenter, see [guide](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-8B3B9004-DE37-4E6B-9AA1-234CDA1BD7F9.html). Trivia, `VSphereCluster` can be only in single VCenter. This will just used as a fallback, if you haven't configured a different identity for a `VSphereMachine``. + +## Examples + +Deploy a `VSphereMachine` with a custom identityRef: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachine +metadata: + name: new-workload-cluster +spec: + server: vcenter + identityRef: + kind: VSphereClusterIdentity + name: identityName +... +``` + +Deploy a `VSphereMachineTemplate` with a custom identityRef: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: new-workload-cluster +spec: + template: + spec: + server: vcenter + identityRef: + kind: VSphereClusterIdentity + name: identityName +... +``` \ No newline at end of file diff --git a/pkg/identity/identity.go b/pkg/identity/identity.go index 712151f8c2..1eec5d3bd3 100644 --- a/pkg/identity/identity.go +++ b/pkg/identity/identity.go @@ -44,9 +44,76 @@ type Credentials struct { Password string } -// GetCredentials returns the VCenter credentials for the VSphereCluster. +func GetCredentialsFromVshpereVM(ctx context.Context, c client.Client, machine *infrav1.VSphereVM, controllerNamespace string) (*Credentials, error) { + if err := validateInputs(c, machine.Namespace, machine.Spec.IdentityRef); err != nil { + return nil, err + } + + ref := machine.Spec.IdentityRef + secret := &corev1.Secret{} + var secretKey client.ObjectKey + + switch ref.Kind { + case infrav1.SecretKind: + secretKey = client.ObjectKey{ + Namespace: machine.Namespace, + Name: ref.Name, + } + case infrav1.VSphereClusterIdentityKind: + identity := &infrav1.VSphereClusterIdentity{} + key := client.ObjectKey{ + Name: ref.Name, + } + if err := c.Get(ctx, key, identity); err != nil { + return nil, err + } + + if !identity.Status.Ready { + return nil, errors.New("identity isn't ready to be used yet") + } + + if identity.Spec.AllowedNamespaces == nil { + return nil, errors.New("allowedNamespaces set to nil, no namespaces are allowed to use this identity") + } + + selector, err := metav1.LabelSelectorAsSelector(&identity.Spec.AllowedNamespaces.Selector) + if err != nil { + return nil, errors.New("failed to build selector") + } + + ns := &corev1.Namespace{} + nsKey := client.ObjectKey{ + Name: machine.Namespace, + } + if err := c.Get(ctx, nsKey, ns); err != nil { + return nil, err + } + if !selector.Matches(labels.Set(ns.GetLabels())) { + return nil, fmt.Errorf("namespace %s is not allowed to use specifified identity", machine.Namespace) + } + + secretKey = client.ObjectKey{ + Name: identity.Spec.SecretName, + Namespace: controllerNamespace, + } + default: + return nil, fmt.Errorf("unknown type %s used for Identity", ref.Kind) + } + + if err := c.Get(ctx, secretKey, secret); err != nil { + return nil, err + } + + credentials := &Credentials{ + Username: getData(secret, UsernameKey), + Password: getData(secret, PasswordKey), + } + + return credentials, nil +} + func GetCredentials(ctx context.Context, c client.Client, cluster *infrav1.VSphereCluster, controllerNamespace string) (*Credentials, error) { - if err := validateInputs(c, cluster); err != nil { + if err := validateInputs(c, cluster.Namespace, cluster.Spec.IdentityRef); err != nil { return nil, err } @@ -113,15 +180,14 @@ func GetCredentials(ctx context.Context, c client.Client, cluster *infrav1.VSphe return credentials, nil } -func validateInputs(c client.Client, cluster *infrav1.VSphereCluster) error { +func validateInputs(c client.Client, namespace string, identityRef *infrav1.VSphereIdentityReference) error { if c == nil { return errors.New("kubernetes client is required") } - if cluster == nil { + if namespace == "" { return errors.New("vsphere cluster is required") } - ref := cluster.Spec.IdentityRef - if ref == nil { + if identityRef == nil { return errors.New("IdentityRef is required") } return nil diff --git a/pkg/identity/identity_test.go b/pkg/identity/identity_test.go index 74145c3d42..7c9a9cba9f 100644 --- a/pkg/identity/identity_test.go +++ b/pkg/identity/identity_test.go @@ -235,19 +235,19 @@ var _ = Describe("validateInputs", func() { Context("If the client is missing", func() { It("should error if client is missing", func() { - Expect(validateInputs(nil, cluster)).NotTo(Succeed()) + Expect(validateInputs(nil, cluster.Namespace, nil)).NotTo(Succeed()) }) }) Context("If the cluster is missing", func() { It("should error if cluster is missing", func() { - Expect(validateInputs(k8sclient, nil)).NotTo(Succeed()) + Expect(validateInputs(k8sclient, "", nil)).NotTo(Succeed()) }) }) Context("If the identityRef is missing on cluster", func() { It("should error if identityRef is missing on cluster", func() { - Expect(validateInputs(k8sclient, cluster)).NotTo(Succeed()) + Expect(validateInputs(k8sclient, cluster.Namespace, nil)).NotTo(Succeed()) }) }) })