From 9e7cce008eacbf052264080690cae56c0d402359 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Tue, 5 Nov 2024 12:57:18 -0500 Subject: [PATCH] api: add oldestKubeletVersion Signed-off-by: Peter Hunt --- api/hypershift/v1beta1/hosted_controlplane.go | 4 + api/hypershift/v1beta1/hostedcluster_types.go | 4 + .../v1beta1/zz_generated.deepcopy.go | 10 + .../MinimumKubeletVersion.yaml | 600 +++++++++++++----- .../MinimumKubeletVersion.yaml | 373 ++++++++--- .../v1beta1/clusterconfiguration.go | 9 + .../hypershift/v1beta1/hostedclusterstatus.go | 9 + .../v1beta1/hostedcontrolplanestatus.go | 9 + .../hostedclusters-CustomNoUpgrade.crd.yaml | 25 + ...stedcontrolplanes-CustomNoUpgrade.crd.yaml | 25 + docs/content/reference/api.md | 22 + .../hypershift/v1beta1/hosted_controlplane.go | 4 + .../hypershift/v1beta1/hostedcluster_types.go | 4 + .../v1beta1/zz_generated.deepcopy.go | 10 + 14 files changed, 884 insertions(+), 224 deletions(-) diff --git a/api/hypershift/v1beta1/hosted_controlplane.go b/api/hypershift/v1beta1/hosted_controlplane.go index a9aa8e81a9..b5825bc7c7 100644 --- a/api/hypershift/v1beta1/hosted_controlplane.go +++ b/api/hypershift/v1beta1/hosted_controlplane.go @@ -318,6 +318,10 @@ type HostedControlPlaneStatus struct { // NodeCount tracks the number of nodes in the HostedControlPlane. NodeCount *int `json:"nodeCount,omitempty"` + + // OldestKubeletVersion tracks the oldest kubelet version in a hosted cluster + // +openshift:enable:FeatureGate=MinimumKubeletVersion + OldestKubeletVersion *string `json:"oldestKubeletVersion,omitempty"` } type APIEndpoint struct { diff --git a/api/hypershift/v1beta1/hostedcluster_types.go b/api/hypershift/v1beta1/hostedcluster_types.go index 209efa02ca..ae224025cf 100644 --- a/api/hypershift/v1beta1/hostedcluster_types.go +++ b/api/hypershift/v1beta1/hostedcluster_types.go @@ -1414,6 +1414,10 @@ type HostedClusterStatus struct { // Platform contains platform-specific status of the HostedCluster // +optional Platform *PlatformStatus `json:"platform,omitempty"` + + // OldestKubeletVersion tracks the oldest kubelet version in a hosted cluster + // +openshift:enable:FeatureGate=MinimumKubeletVersion + OldestKubeletVersion *string `json:"oldestKubeletVersion,omitempty"` } // PlatformStatus contains platform-specific status diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index cd06a4e580..935f9f2880 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -1443,6 +1443,11 @@ func (in *HostedClusterStatus) DeepCopyInto(out *HostedClusterStatus) { *out = new(PlatformStatus) (*in).DeepCopyInto(*out) } + if in.OldestKubeletVersion != nil { + in, out := &in.OldestKubeletVersion, &out.OldestKubeletVersion + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostedClusterStatus. @@ -1656,6 +1661,11 @@ func (in *HostedControlPlaneStatus) DeepCopyInto(out *HostedControlPlaneStatus) *out = new(int) **out = **in } + if in.OldestKubeletVersion != nil { + in, out := &in.OldestKubeletVersion, &out.OldestKubeletVersion + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostedControlPlaneStatus. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/MinimumKubeletVersion.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/MinimumKubeletVersion.yaml index ff8968fb88..baefced550 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/MinimumKubeletVersion.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/MinimumKubeletVersion.yaml @@ -73,8 +73,11 @@ spec: properties: additionalTrustBundle: description: |- - AdditionalTrustBundle is a reference to a ConfigMap containing a - PEM-encoded X.509 certificate bundle that will be added to the hosted controlplane and nodes + additionalTrustBundle is a local reference to a ConfigMap that must have a "ca-bundle.crt" key + whose content must be a PEM-encoded X.509 certificate bundle that will be added to the hosted controlplane and nodes + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. + This will be part of every payload generated by the controllers for any NodePool of the HostedCluster. + Changing this value will trigger a rollout for all existing NodePools in the cluster. properties: name: default: "" @@ -109,34 +112,35 @@ spec: x-kubernetes-map-type: atomic autoscaling: description: |- - Autoscaling specifies auto-scaling behavior that applies to all NodePools - associated with the control plane. + autoscaling specifies auto-scaling behavior that applies to all NodePools + associated with this HostedCluster. properties: maxNodeProvisionTime: description: |- - MaxNodeProvisionTime is the maximum time to wait for node provisioning + maxNodeProvisionTime is the maximum time to wait for node provisioning before considering the provisioning to be unsuccessful, expressed as a Go duration string. The default is 15 minutes. pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ type: string maxNodesTotal: description: |- - MaxNodesTotal is the maximum allowable number of nodes across all NodePools - for a HostedCluster. The autoscaler will not grow the cluster beyond this + maxNodesTotal is the maximum allowable number of nodes for the Autoscaler scale out to be operational. + The autoscaler will not grow the cluster beyond this number. + If omitted, the autoscaler will not have a maximum limit. number. format: int32 minimum: 0 type: integer maxPodGracePeriod: description: |- - MaxPodGracePeriod is the maximum seconds to wait for graceful pod + maxPodGracePeriod is the maximum seconds to wait for graceful pod termination before scaling down a NodePool. The default is 600 seconds. format: int32 minimum: 0 type: integer podPriorityThreshold: description: |- - PodPriorityThreshold enables users to schedule "best-effort" pods, which + podPriorityThreshold enables users to schedule "best-effort" pods, which shouldn't trigger autoscaler actions, but only run when there are spare resources available. The default is -10. @@ -147,23 +151,27 @@ spec: type: object channel: description: |- - channel is an identifier for explicitly requesting that a non-default - set of updates be applied to this cluster. The default channel will be - contain stable updates that are appropriate for production clusters. + channel is an identifier for explicitly requesting that a non-default set of updates be applied to this cluster. + If ommited no particular upgrades are suggested. + maxLength: 100 + minLength: 1 type: string clusterID: description: |- - ClusterID uniquely identifies this cluster. This is expected to be - an RFC4122 UUID value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx in - hexadecimal values). - As with a Kubernetes metadata.uid, this ID uniquely identifies this - cluster in space and time. - This value identifies the cluster in metrics pushed to telemetry and - metrics produced by the control plane operators. If a value is not - specified, an ID is generated. After initial creation, the value is - immutable. - pattern: '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}' + clusterID uniquely identifies this cluster. This is expected to be an RFC4122 UUID value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx in hexadecimal digits). + As with a Kubernetes metadata.uid, this ID uniquely identifies this cluster in space and time. + This value identifies the cluster in metrics pushed to telemetry and metrics produced by the control plane operators. + If a value is not specified, a random clusterID will be generated and set by the controller. + Once set, this value is immutable. + maxLength: 36 + minLength: 36 type: string + x-kubernetes-validations: + - message: clusterID must be an RFC4122 UUID value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + in hexadecimal digits) + rule: self.matches('[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') + - message: clusterID is immutable + rule: oldSelf == "" || self == oldSelf configuration: description: |- Configuration specifies configuration for individual OCP components in the @@ -714,6 +722,7 @@ spec: registries, and policies to block or allow registry hostnames. When exposing OpenShift's image registry to the public, this also lets cluster admins specify the external hostname. + Changing this value will trigger a rollout for all existing NodePools in the cluster. properties: additionalTrustedCA: description: |- @@ -1218,6 +1227,27 @@ spec: - v2 - "" type: string + minimumKubeletVersion: + description: |- + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 + type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible + format of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating @@ -1920,8 +1950,7 @@ spec: type: array type: object proxy: - description: Proxy holds cluster-wide information on how to configure - default proxies for the cluster. + description: ProxySpec contains cluster proxy creation configuration. properties: httpProxy: description: httpProxy is the URL of the proxy for HTTP requests. Empty @@ -2048,48 +2077,88 @@ spec: type: object controlPlaneRelease: description: |- - ControlPlaneRelease specifies the desired OCP release payload for - control plane components running on the management cluster. - Updating this field will trigger a rollout of the control plane. The - behavior of the rollout will be driven by the ControllerAvailabilityPolicy - and InfrastructureAvailabilityPolicy. - If not defined, Release is used + controlPlaneRelease is like spec.release but only for the components running on the management cluster. + This excludes any operand which will land in the hosted cluster data plane. + It is useful when you need to apply patch management side like a CVE, transparently for the hosted cluster. + Version input for this field is free, no validation is performed against spec.release or maximum and minimum is performed. + If defined, it will dicate the version of the components running management side, while spec.release will dictate the version of the components landing in the hosted cluster data plane. + If not defined, spec.release is used for both. + Changing this field will trigger a rollout of the control plane. + The behavior of the rollout will be driven by the ControllerAvailabilityPolicy and InfrastructureAvailabilityPolicy for PDBs and maxUnavailable and surce policies. properties: image: - description: Image is the image pullspec of an OCP release payload - image. - pattern: ^(\w+\S+)$ + description: |- + Image is the image pullspec of an OCP release payload image. + See https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags for a list of available images. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: Image must start with a word character (letters, digits, + or underscores) and contain no white spaces + rule: self.matches('^(\\w+\\S+)$') required: - image type: object controllerAvailabilityPolicy: default: HighlyAvailable description: |- - ControllerAvailabilityPolicy specifies the availability policy applied to - critical control plane components. The default value is HighlyAvailable. + controllerAvailabilityPolicy specifies the availability policy applied to critical control plane components like the Kube API Server. + Possible values are HighlyAvailable and SingleReplica. The default value is HighlyAvailable. + enum: + - HighlyAvailable + - SingleReplica type: string dns: - description: DNS specifies DNS configuration for the cluster. + description: dns specifies the DNS configuration for the hosted cluster + ingress. properties: baseDomain: - description: BaseDomain is the base domain of the cluster. + description: |- + baseDomain is the base domain of the hosted cluster. + It will be used to confgure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is ommitted, the hostedCluster.name will be used as the subdomain. + Once set, this field is immutable. + When the value is the empty string "", the controller might default to a value depending on the platform. + maxLength: 253 type: string + x-kubernetes-validations: + - message: baseDomain is immutable + rule: oldSelf == "" || self == oldSelf baseDomainPrefix: description: |- - BaseDomainPrefix is the base domain prefix of the cluster. - defaults to clusterName if not set. Set it to "" if you don't want a prefix to be prepended to BaseDomain. + baseDomainPrefix is the base domain prefix for the hosted cluster ingress. + It will be used to confgure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is ommitted, the hostedCluster.name will be used as the subdomain. + Set baseDomainPrefix to an empty string "", if you don't want a prefix at all (not even hostedCluster.name) to be prepended to baseDomain. + This field is immutable. + maxLength: 253 type: string + x-kubernetes-validations: + - message: baseDomainPrefix is immutable + rule: self == oldSelf privateZoneID: description: |- - PrivateZoneID is the Hosted Zone ID where all the DNS records that are only - available internally to the cluster exist. + privateZoneID is the Hosted Zone ID where all the DNS records that are only available internally to the cluster exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: privateZoneID is immutable + rule: oldSelf == "" || self == oldSelf publicZoneID: description: |- - PublicZoneID is the Hosted Zone ID where all the DNS records that are - publicly accessible to the internet exist. + publicZoneID is the Hosted Zone ID where all the DNS records that are publicly accessible to the internet exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: publicZoneID is immutable + rule: oldSelf == "" || self == oldSelf required: - baseDomain type: object @@ -2102,20 +2171,20 @@ spec: type: PersistentVolume managementType: Managed description: |- - Etcd specifies configuration for the control plane etcd cluster. The - default ManagementType is Managed. Once set, the ManagementType cannot be + etcd specifies configuration for the control plane etcd cluster. The + default managementType is Managed. Once set, the managementType cannot be changed. properties: managed: - description: Managed specifies the behavior of an etcd cluster + description: managed specifies the behavior of an etcd cluster managed by HyperShift. properties: storage: - description: Storage specifies how etcd data is persisted. + description: storage specifies how etcd data is persisted. properties: persistentVolume: description: |- - PersistentVolume is the configuration for PersistentVolume etcd storage. + persistentVolume is the configuration for PersistentVolume etcd storage. With this implementation, a PersistentVolume will be allocated for every etcd member (either 1 or 3 depending on the HostedCluster control plane availability configuration). @@ -2125,8 +2194,10 @@ spec: - type: integer - type: string default: 8Gi - description: Size is the minimum size of the data - volume for each etcd member. + description: |- + size is the minimum size of the data volume for each etcd member. + Default is 8Gi. + This field is immutable pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true x-kubernetes-validations: @@ -2134,14 +2205,16 @@ spec: rule: self == oldSelf storageClassName: description: |- - StorageClassName is the StorageClass of the data volume for each etcd member. - + storageClassName is the StorageClass of the data volume for each etcd member. See https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1. type: string + x-kubernetes-validations: + - message: storageClassName is immutable + rule: self == oldSelf type: object restoreSnapshotURL: description: |- - RestoreSnapshotURL allows an optional URL to be provided where + restoreSnapshotURL allows an optional URL to be provided where an etcd snapshot can be downloaded, for example a pre-signed URL referencing a storage service. This snapshot will be restored on initial startup, only when the etcd PV @@ -2154,8 +2227,9 @@ spec: 1 entry rule: self.size() <= 1 type: - description: Type is the kind of persistent storage implementation - to use for etcd. + description: |- + type is the kind of persistent storage implementation to use for etcd. + Only PersistentVolume is supported at the moment. enum: - PersistentVolume type: string @@ -2166,19 +2240,25 @@ spec: - storage type: object managementType: - description: ManagementType defines how the etcd cluster is managed. + description: |- + managementType defines how the etcd cluster is managed. + This can be either Managed or Unmanaged. + This field is immutable. enum: - Managed - Unmanaged type: string + x-kubernetes-validations: + - message: managementType is immutable + rule: self == oldSelf unmanaged: description: |- - Unmanaged specifies configuration which enables the control plane to - integrate with an eternally managed etcd cluster. + unmanaged specifies configuration which enables the control plane to + integrate with an externally managed etcd cluster. properties: endpoint: description: |- - Endpoint is the full etcd cluster client endpoint URL. For example: + endpoint is the full etcd cluster client endpoint URL. For example: https://etcd-client:2379 @@ -2186,7 +2266,7 @@ spec: pattern: ^https:// type: string tls: - description: TLS specifies TLS configuration for HTTPS etcd + description: tls specifies TLS configuration for HTTPS etcd client endpoints. properties: clientSecret: @@ -2219,16 +2299,31 @@ spec: required: - managementType type: object + x-kubernetes-validations: + - message: Only managed configuration must be set when managementType + is Managed + rule: 'self.managementType == ''Managed'' ? has(self.managed) : + !has(self.managed)' + - message: Only unmanaged configuration must be set when managementType + is Unmanaged + rule: 'self.managementType == ''Unmanaged'' ? has(self.unmanaged) + : !has(self.unmanaged)' fips: description: |- - FIPS indicates whether this cluster's nodes will be running in FIPS mode. + fips indicates whether this cluster's nodes will be running in FIPS mode. If set to true, the control plane's ignition server will be configured to expect that nodes joining the cluster will be FIPS-enabled. type: boolean + x-kubernetes-validations: + - message: fips is immutable + rule: self == oldSelf imageContentSources: description: |- - ImageContentSources specifies image mirrors that can be used by cluster + imageContentSources specifies image mirrors that can be used by cluster nodes to pull content. + When imageContentSources is set, the controllers will generate a machineConfig. + This MachineConfig will be part of every payload generated by the controllers for any NodePool of the HostedCluster. + Changing this value will trigger a rollout for all existing NodePools in the cluster. items: description: |- ImageContentSource specifies image mirrors that can be used by cluster nodes @@ -2253,26 +2348,49 @@ spec: type: array infraID: description: |- - InfraID is a globally unique identifier for the cluster. This identifier - will be used to associate various cloud resources with the HostedCluster - and its associated NodePools. + infraID is a globally unique identifier for the cluster. + It must consist of lowercase alphanumeric characters and hyphens ('-') only, and start and end with an alphanumeric character. + It must be no more than 253 characters in length. + This identifier will be used to associate various cloud resources with the HostedCluster and its associated NodePools. + infraID is used to compute and tag created resources with "kubernetes.io/cluster/"+hcluster.Spec.InfraID which has contractual meaning for the cloud provider implementations. + If a value is not specified, a random infraID will be generated and set by the controller. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: infraID must consist of lowercase alphanumeric characters + or '-', start and end with an alphanumeric character, and be between + 1 and 253 characters + rule: self.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + - message: infraID is immutable + rule: oldSelf == "" || self == oldSelf infrastructureAvailabilityPolicy: default: SingleReplica description: |- - InfrastructureAvailabilityPolicy specifies the availability policy applied - to infrastructure services which run on cluster nodes. The default value is - SingleReplica. + infrastructureAvailabilityPolicy specifies the availability policy applied to infrastructure services which run on the hosted cluster data plane like the ingress controller and image registry controller. + Possible values are HighlyAvailable and SingleReplica. The default value is SingleReplica. + enum: + - HighlyAvailable + - SingleReplica type: string issuerURL: default: https://kubernetes.default.svc description: |- - IssuerURL is an OIDC issuer URL which is used as the issuer in all - ServiceAccount tokens generated by the control plane API server. The - default value is kubernetes.default.svc, which only works for in-cluster + issuerURL is an OIDC issuer URL which will be used as the issuer in all + ServiceAccount tokens generated by the control plane API server via --service-account-issuer kube api server flag. + https://k8s-docs.netlify.app/en/docs/reference/command-line-tools-reference/kube-apiserver/ + https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#serviceaccount-token-volume-projection + The default value is kubernetes.default.svc, which only works for in-cluster validation. - format: uri + If the platform is AWS and this value is set, the controller will update an s3 object with the appropriate OIDC documents (using the serviceAccountSigningKey info) into that issuerURL. + The expectation is for this s3 url to be backed by an OIDC provider in the AWS IAM. type: string + x-kubernetes-validations: + - message: issuerURL is immutable + rule: self == oldSelf + - message: issuerURL must be a valid absolute URL + rule: isURL(self) networking: default: clusterNetwork: @@ -2280,23 +2398,29 @@ spec: networkType: OVNKubernetes serviceNetwork: - cidr: 172.31.0.0/16 - description: Networking specifies network configuration for the cluster. + description: |- + networking specifies network configuration for the hosted cluster. + Defaults to OVNKubernetes with a cluster network of cidr: "10.132.0.0/14" and a service network of cidr: "172.31.0.0/16". properties: apiServer: description: |- - APIServer contains advanced network settings for the API server that affect - how the APIServer is exposed inside a cluster node. + apiServer contains advanced network settings for the API server that affect + how the APIServer is exposed inside a hosted cluster node. properties: advertiseAddress: description: |- - AdvertiseAddress is the address that nodes will use to talk to the API + advertiseAddress is the address that pods within the nodes will use to talk to the API server. This is an address associated with the loopback adapter of each node. If not specified, the controller will take default values. The default values will be set as 172.20.0.1 or fd00::1. + This value is immutable. type: string + x-kubernetes-validations: + - message: advertiseAddress is immutable + rule: self == oldSelf allowedCIDRBlocks: description: |- - AllowedCIDRBlocks is an allow list of CIDR blocks that can access the APIServer + allowedCIDRBlocks is an allow list of CIDR blocks that can access the APIServer If not specified, traffic is allowed from all addresses. This depends on underlying support by the cloud provider for Service LoadBalancerSourceRanges items: @@ -2305,42 +2429,64 @@ spec: type: array port: description: |- - Port is the port at which the APIServer is exposed inside a node. Other + port is the port at which the APIServer is exposed inside a node. Other pods using host networking cannot listen on this port. - If unset 6443 is used. + If ommited 6443 is used. This is useful to choose a port other than the default one which might interfere with customer environments e.g. https://github.com/openshift/hypershift/pull/356. Setting this to 443 is possible only for backward compatibility reasons and it's discouraged. Doing so, it would result in the controller overriding the KAS endpoint in the guest cluster having a discrepancy with the KAS Pod and potentially causing temporarily network failures. + This value is immutable. format: int32 type: integer + x-kubernetes-validations: + - message: port is immutable + rule: self == oldSelf type: object clusterNetwork: default: - cidr: 10.132.0.0/14 - description: ClusterNetwork is the list of IP address pools for - pods. + description: |- + clusterNetwork is the list of IP address pools for pods. + Defaults to cidr: "10.132.0.0/14". + Currently only one entry is supported. + This field is immutable. items: description: |- ClusterNetworkEntry is a single IP address block for pod IP blocks. IP blocks are allocated with size 2^HostSubnetLength. properties: cidr: - description: CIDR is the IP block address pool. + description: cidr is the IP block address pool. + maxLength: 43 type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') hostPrefix: description: |- - HostPrefix is the prefix size to allocate to each node from the CIDR. - For example, 24 would allocate 2^8=256 adresses to each node. If this + hostPrefix is the prefix size to allocate to each node from the CIDR. + For example, 24 would allocate 2^(32-24)=2^8=256 adresses to each node. If this field is not used by the plugin, it can be left unset. format: int32 type: integer required: - cidr type: object + maxItems: 2 + minItems: 1 type: array + x-kubernetes-validations: + - message: clusterNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf machineNetwork: - description: MachineNetwork is the list of IP address pools for - machines. + description: |- + machineNetwork is the list of IP address pools for machines. + This might be used among other things to generate appropriate networking security groups in some clouds providers. + Currently only one entry or two for dual stack is supported. + This field is immutable. items: description: MachineNetworkEntry is a single IP address block for node IP blocks. @@ -2348,15 +2494,30 @@ spec: cidr: description: CIDR is the IP block address pool for machines within the cluster. + maxLength: 43 type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') required: - cidr type: object + maxItems: 2 + minItems: 1 type: array + x-kubernetes-validations: + - message: machineNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf networkType: default: OVNKubernetes - description: NetworkType specifies the SDN provider used for cluster - networking. + description: |- + networkType specifies the SDN provider used for cluster networking. + Defaults to OVNKubernetes. + This field is required and immutable. + kubebuilder:validation:XValidation:rule="self == oldSelf", message="networkType is immutable" enum: - OpenShiftSDN - Calico @@ -2367,30 +2528,55 @@ spec: default: - cidr: 172.31.0.0/16 description: |- - ServiceNetwork is the list of IP address pools for services. - NOTE: currently only one entry is supported. + serviceNetwork is the list of IP address pools for services. + Defaults to cidr: "172.31.0.0/16". + Currently only one entry is supported. + This field is immutable. items: description: ServiceNetworkEntry is a single IP address block for the service network. properties: cidr: - description: CIDR is the IP block address pool for services - within the cluster. + description: cidr is the IP block address pool for services + within the cluster in CIDR format (e.g., 192.168.1.0/24 + or 2001:0db8::/64) + maxLength: 43 type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') required: - cidr type: object + maxItems: 2 + minItems: 1 type: array - required: - - clusterNetwork - - networkType + x-kubernetes-validations: + - message: serviceNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf type: object + x-kubernetes-validations: + - message: CIDR ranges in machineNetwork, clusterNetwork, and serviceNetwork + must be unique and non-overlapping + rule: (!has(self.machineNetwork) && self.clusterNetwork.all(c, self.serviceNetwork.all(s, + c.cidr != s.cidr)) || (has(self.machineNetwork) && (self.machineNetwork.all(m, + self.clusterNetwork.all(c, m.cidr != c.cidr)) && self.machineNetwork.all(m, + self.serviceNetwork.all(s, m.cidr != s.cidr)) && self.clusterNetwork.all(c, + self.serviceNetwork.all(s, c.cidr != s.cidr))))) nodeSelector: additionalProperties: type: string - description: NodeSelector when specified, must be true for the pods - managed by the HostedCluster to be scheduled. + description: |- + NodeSelector when specified, is propagated to all control plane Deployments and Stateful sets running management side. + It must be satisfied by the management Nodes for the pods to be scheduled. Otherwise the HostedCluster will enter a degraded state. + Changes to this field will propagate to existing Deployments and StatefulSets. type: object + x-kubernetes-validations: + - message: nodeSelector map can have at most 20 entries + rule: size(self) <= 20 olmCatalogPlacement: default: management description: |- @@ -2407,14 +2593,21 @@ spec: rule: self == oldSelf pausedUntil: description: |- - PausedUntil is a field that can be used to pause reconciliation on a resource. - Either a date can be provided in RFC3339 format or a boolean. If a date is + pausedUntil is a field that can be used to pause reconciliation on the HostedCluster controller, resulting in any change to the HostedCluster being ignored. + Either a date can be provided in RFC3339 format or a boolean as in 'true', 'false', 'True', 'False'. If a date is provided: reconciliation is paused on the resource until that date. If the boolean true is provided: reconciliation is paused on the resource until the field is removed. + maxLength: 35 + minLength: 4 type: string + x-kubernetes-validations: + - message: PausedUntil must be a date in RFC3339 format or 'True', + 'true', 'False' or 'false' + rule: self.matches('^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.*$') + || self in ['true', 'false', 'True', 'False'] platform: description: |- - Platform specifies the underlying infrastructure provider for the cluster + platform specifies the underlying infrastructure provider for the cluster and is used to configure platform specific behavior. properties: agent: @@ -2915,13 +3108,43 @@ spec: rule: self == oldSelf subnetID: description: |- - SubnetID is the subnet ID of an existing subnet where the load balancer for node egress will be created. This - subnet is expected to be a subnet within the VNET specified in VnetID. This subnet is expected to exist under the - same subscription as SubscriptionID. - - In ARO HCP, managed services will create the aforementioned load balancer in ResourceGroupName. + subnetID is the subnet ID of an existing subnet where the nodes in the nodepool will be created. This can be a + different subnet than the one listed in the HostedCluster, HostedCluster.Spec.Platform.Azure.SubnetID, but must + exist in the same network, HostedCluster.Spec.Platform.Azure.VnetID, and must exist under the same subscription ID, + HostedCluster.Spec.Platform.Azure.SubscriptionID. + subnetID is immutable once set. + The subnetID should be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}`. + The subscriptionId in the encryptionSetID must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + The resourceGroupName should be between 1 and 90 characters, consisting only of alphanumeric characters, hyphens, underscores, periods and paranthesis and must not end with a period (.) character. + The vnetName should be between 2 and 64 characters, consisting only of alphanumeric characters, hyphens, underscores and periods and must not end with either a period (.) or hyphen (-) character. + The subnetName should be between 1 and 80 characters, consisting only of alphanumeric characters, hyphens and underscores and must start with an alphanumeric character and must not end with a period (.) or hyphen (-) character. + maxLength: 355 + minLength: 1 type: string x-kubernetes-validations: + - message: encryptionSetID must be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}` + rule: size(self.split('/')) == 11 && self.matches('^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Network/virtualNetworks/.*/subnets/.*$') + - message: The resourceGroupName should be between 1 and 90 + characters, consisting only of alphanumeric characters, + hyphens, underscores, periods and paranthesis + rule: self.split('/')[4].matches('[a-zA-Z0-9-_\\(\\)\\.]{1,90}') + - message: the resourceGroupName in the subnetID must not + end with a period (.) character + rule: '!self.split(''/'')[4].endsWith(''.'')' + - message: The vnetName should be between 2 and 64 characters, + consisting only of alphanumeric characters, hyphens, underscores + and periods + rule: self.split('/')[8].matches('[a-zA-Z0-9-_\\.]{2,64}') + - message: the vnetName in the subnetID must not end with + either a period (.) or hyphen (-) character + rule: '!self.split(''/'')[8].endsWith(''.'') && !self.split(''/'')[8].endsWith(''-'')' + - message: The subnetName should be between 1 and 80 characters, + consisting only of alphanumeric characters, hyphens and + underscores and must start with an alphanumeric character + rule: self.split('/')[10].matches('[a-zA-Z0-9][a-zA-Z0-9-_\\.]{0,79}') + - message: the subnetName in the subnetID must not end with + a period (.) or hyphen (-) character + rule: '!self.split(''/'')[10].endsWith(''.'') && !self.split(''/'')[10].endsWith(''-'')' - message: SubnetID is immutable rule: self == oldSelf subscriptionID: @@ -3329,9 +3552,12 @@ spec: type: object pullSecret: description: |- - PullSecret references a pull secret to be injected into the container - runtime of all cluster nodes. The secret must have a key named - ".dockerconfigjson" whose value is the pull secret JSON. + pullSecret is a local reference to a Secret that must have a ".dockerconfigjson" key whose content must be a valid Openshift pull secret JSON. + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. + This pull secret will be part of every payload generated by the controllers for any NodePool of the HostedCluster + and it will be injected into the container runtime of all NodePools. + Changing this value will trigger a rollout for all existing NodePools in the cluster. + Changing the content of the secret inplace will not trigger a rollout and might result in unpredicatble behaviour. properties: name: default: "" @@ -3346,23 +3572,31 @@ spec: x-kubernetes-map-type: atomic release: description: |- - Release specifies the desired OCP release payload for the hosted cluster. - - Updating this field will trigger a rollout of the control plane. The - behavior of the rollout will be driven by the ControllerAvailabilityPolicy - and InfrastructureAvailabilityPolicy. + release specifies the desired OCP release payload for all the hosted cluster components. + This includes those components running management side like the Kube API Server and the CVO but also the operands which land in the hosted cluster data plane like the ingress controller, ovn agents, etc. + The maximum and minimum supported release versions are determined by the running Hypersfhit Operator. + Attempting to use an unsupported version will result in the HostedCluster being degraded and the validateReleaseImage condition being false. + Attempting to use a release with a skew against a NodePool release bigger than N-2 for the y-stream will result in leaving the NodePool in an unsupported state. + Changing this field will trigger a rollout of the control plane components. + The behavior of the rollout will be driven by the ControllerAvailabilityPolicy and InfrastructureAvailabilityPolicy for PDBs and maxUnavailable and surce policies. properties: image: - description: Image is the image pullspec of an OCP release payload - image. - pattern: ^(\w+\S+)$ + description: |- + Image is the image pullspec of an OCP release payload image. + See https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags for a list of available images. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: Image must start with a word character (letters, digits, + or underscores) and contain no white spaces + rule: self.matches('^(\\w+\\S+)$') required: - image type: object secretEncryption: description: |- - SecretEncryption specifies a Kubernetes secret encryption strategy for the + secretEncryption specifies a Kubernetes secret encryption strategy for the control plane. properties: aescbc: @@ -3635,11 +3869,12 @@ spec: type: object serviceAccountSigningKey: description: |- - ServiceAccountSigningKey is a reference to a secret containing the private key - used by the service account token issuer. The secret is expected to contain - a single key named "key". If not specified, a service account signing key will - be generated automatically for the cluster. When specifying a service account - signing key, a IssuerURL must also be specified. + serviceAccountSigningKey is a local reference to a secret that must have a "key" key whose content must be the private key + used by the service account token issuer. + If not specified, a service account signing key will + be generated automatically for the cluster. + When specifying a service account signing key, an IssuerURL must also be specified. + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. properties: name: default: "" @@ -3654,18 +3889,27 @@ spec: x-kubernetes-map-type: atomic services: description: |- - Services specifies how individual control plane services are published from - the hosting cluster of the control plane. + services specifies how individual control plane services endpoints are published for consumption. + This requires APIServer;OAuthServer;Konnectivity;Ignition. + This field is immutable for all platforms but IBMCloud. + Max is 6 to account for OIDC;OVNSbDb for backward compability though they are no-op. - If a given service is not present in this list, it will be exposed publicly - by default. + -kubebuilder:validation:XValidation:rule="self.all(s, !(s.service == 'APIServer' && s.servicePublishingStrategy.type == 'Route') || has(s.servicePublishingStrategy.route.hostname))",message="If serviceType is 'APIServer' and publishing strategy is 'Route', then hostname must be set" + -kubebuilder:validation:XValidation:rule="['APIServer', 'OAuthServer', 'Konnectivity', 'Ignition'].all(requiredType, self.exists(s, s.service == requiredType))",message="Services list must contain at least 'APIServer', 'OAuthServer', 'Konnectivity', and 'Ignition' service types" + -kubebuilder:validation:XValidation:rule="self.filter(s, s.servicePublishingStrategy.type == 'Route' && has(s.servicePublishingStrategy.route) && has(s.servicePublishingStrategy.route.hostname)).all(x, self.filter(y, y.servicePublishingStrategy.type == 'Route' && (has(y.servicePublishingStrategy.route) && has(y.servicePublishingStrategy.route.hostname) && y.servicePublishingStrategy.route.hostname == x.servicePublishingStrategy.route.hostname)).size() <= 1)",message="Each route publishingStrategy 'hostname' must be unique within the Services list." + -kubebuilder:validation:XValidation:rule="self.filter(s, s.servicePublishingStrategy.type == 'NodePort' && has(s.servicePublishingStrategy.nodePort) && has(s.servicePublishingStrategy.nodePort.address) && has(s.servicePublishingStrategy.nodePort.port)).all(x, self.filter(y, y.servicePublishingStrategy.type == 'NodePort' && (has(y.servicePublishingStrategy.nodePort) && has(y.servicePublishingStrategy.nodePort.address) && y.servicePublishingStrategy.nodePort.address == x.servicePublishingStrategy.nodePort.address && has(y.servicePublishingStrategy.nodePort.port) && y.servicePublishingStrategy.nodePort.port == x.servicePublishingStrategy.nodePort.port )).size() <= 1)",message="Each nodePort publishingStrategy 'nodePort' and 'hostname' must be unique within the Services list." items: description: |- - ServicePublishingStrategyMapping specifies how individual control plane - services are published from the hosting cluster of a control plane. + ServicePublishingStrategyMapping specifies how individual control plane services endpoints are published for consumption. + This includes APIServer;OAuthServer;Konnectivity;Ignition. + If a given service is not present in this list, it will be exposed publicly by default. properties: service: - description: Service identifies the type of service being published. + description: |- + service identifies the type of service being published. + It can be APIServer;OAuthServer;Konnectivity;Ignition + OVNSbDb;OIDC are no-op and kept for backward compatibility. + This field is immutable. enum: - APIServer - OAuthServer @@ -3675,29 +3919,44 @@ spec: - OVNSbDb type: string servicePublishingStrategy: - description: ServicePublishingStrategy specifies how to publish - Service. + description: servicePublishingStrategy specifies how to publish + a service endpoint. properties: loadBalancer: - description: LoadBalancer configures exposing a service - using a LoadBalancer. + description: loadBalancer configures exposing a service + using a dedicated LoadBalancer. properties: hostname: - description: Hostname is the name of the DNS record - that will be created pointing to the LoadBalancer. + description: |- + hostname is the name of the DNS record that will be created pointing to the LoadBalancer and passed through to consumers of the service. + If ommited, the value will be infered from the corev1.Service Load balancer type .status. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: baseDomain must be a valid base domain (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') type: object nodePort: - description: NodePort configures exposing a service using + description: nodePort configures exposing a service using a NodePort. properties: address: - description: Address is the host/ip that the NodePort + description: address is the host/ip that the NodePort service is exposed over. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: address must be a valid hostname, IPv4, or + IPv6 address + rule: self.matches('^(([a-zA-Z0-9][-a-zA-Z0-9]*\\.)+[a-zA-Z]{2,}|localhost)$') + || self.matches('^((\\d{1,3}\\.){3}\\d{1,3})$') + || self.matches('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$') port: description: |- - Port is the port of the NodePort service. If <=0, the port is dynamically + port is the port of the NodePort service. If <=0, the port is dynamically assigned when the service is created. format: int32 type: integer @@ -3705,17 +3964,26 @@ spec: - address type: object route: - description: Route configures exposing a service using a - Route. + description: |- + route configures exposing a service using a Route through and an ingress controller behind a cloud Load Balancer. + The specifics of the setup are platform dependent. properties: hostname: - description: Hostname is the name of the DNS record - that will be created pointing to the Route. + description: |- + Hostname is the name of the DNS record that will be created pointing to the Route and passed through to consumers of the service. + If ommited, the value will be infered from management ingress.Spec.Domain. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: hostname must be a valid domain name (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') type: object type: - description: Type is the publishing strategy used for the - service. + description: |- + type is the publishing strategy used for the service. + It can be LoadBalancer;NodePort;Route;None;S3 enum: - LoadBalancer - NodePort @@ -3726,16 +3994,40 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: nodePort is required when type is NodePort, and forbidden + otherwise + rule: 'self.type == ''NodePort'' ? has(self.nodePort) : !has(self.nodePort)' + - message: only route is allowed when type is Route, and forbidden + otherwise + rule: 'self.type == ''Route'' ? !has(self.nodePort) && !has(self.loadBalancer) + : !has(self.route)' + - message: only loadBalancer is required when type is LoadBalancer, + and forbidden otherwise + rule: 'self.type == ''LoadBalancer'' ? !has(self.nodePort) + && !has(self.route) : !has(self.loadBalancer)' + - message: None does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''None'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' + - message: S3 does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''S3'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' required: - service - servicePublishingStrategy type: object + maxItems: 6 + minItems: 4 type: array sshKey: description: |- - SSHKey references an SSH key to be injected into all cluster node sshd - servers. The secret must have a single key "id_rsa.pub" whose value is the - public part of an SSH key. + sshKey is a local reference to a Secret that must have a "id_rsa.pub" key whose content must be the public part of 1..N SSH keys. + If the reference is set but none of the above requirements are met, the HostedCluster will enter a degraded state. + When sshKey is set, the controllers will generate a machineConfig with the sshAuthorizedKeys https://coreos.github.io/ignition/configuration-v3_2/ populated with this value. + This MachineConfig will be part of every payload generated by the controllers for any NodePool of the HostedCluster. + Changing this value will trigger a rollout for all existing NodePools in the cluster. properties: name: default: "" @@ -3791,15 +4083,19 @@ spec: updateService: description: |- updateService may be used to specify the preferred upstream update service. - By default it will use the appropriate update service for the cluster and region. + If ommitted we will use the appropriate update service for the cluster and region. + This is used by the control plane operator to determine and signal the appropriate available upgrades in the hostedCluster.status. type: string + x-kubernetes-validations: + - message: updateService must be a valid absolute URL + rule: isURL(self) required: + - etcd - networking - platform - pullSecret - release - services - - sshKey type: object x-kubernetes-validations: - message: Services is immutable. Changes might result in unpredictable @@ -3826,6 +4122,8 @@ spec: rule: 'self.platform.type == "Azure" ? self.services.exists(s, s.service == "Ignition" && s.servicePublishingStrategy.type == "Route" && s.servicePublishingStrategy.route.hostname != "") : true' + - message: If serviceAccountSigningKey is set, issuerURL must be set + rule: has(self.issuerURL) || !has(self.serviceAccountSigningKey) status: description: Status is the latest observed status of the HostedCluster. properties: @@ -3952,6 +4250,10 @@ spec: with the name of an identity provider defined on the HostedCluster. This is populated after the infrastructure is ready. type: string + oldestKubeletVersion: + description: OldestKubeletVersion tracks the oldest kubelet version + in a hosted cluster + type: string payloadArch: description: |- payloadArch represents the CPU architecture type of the HostedCluster.Spec.Release.Image. The valid values are: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/MinimumKubeletVersion.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/MinimumKubeletVersion.yaml index f464f34003..4799346490 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/MinimumKubeletVersion.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/MinimumKubeletVersion.yaml @@ -85,29 +85,30 @@ spec: properties: maxNodeProvisionTime: description: |- - MaxNodeProvisionTime is the maximum time to wait for node provisioning + maxNodeProvisionTime is the maximum time to wait for node provisioning before considering the provisioning to be unsuccessful, expressed as a Go duration string. The default is 15 minutes. pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ type: string maxNodesTotal: description: |- - MaxNodesTotal is the maximum allowable number of nodes across all NodePools - for a HostedCluster. The autoscaler will not grow the cluster beyond this + maxNodesTotal is the maximum allowable number of nodes for the Autoscaler scale out to be operational. + The autoscaler will not grow the cluster beyond this number. + If omitted, the autoscaler will not have a maximum limit. number. format: int32 minimum: 0 type: integer maxPodGracePeriod: description: |- - MaxPodGracePeriod is the maximum seconds to wait for graceful pod + maxPodGracePeriod is the maximum seconds to wait for graceful pod termination before scaling down a NodePool. The default is 600 seconds. format: int32 minimum: 0 type: integer podPriorityThreshold: description: |- - PodPriorityThreshold enables users to schedule "best-effort" pods, which + podPriorityThreshold enables users to schedule "best-effort" pods, which shouldn't trigger autoscaler actions, but only run when there are spare resources available. The default is -10. @@ -678,6 +679,7 @@ spec: registries, and policies to block or allow registry hostnames. When exposing OpenShift's image registry to the public, this also lets cluster admins specify the external hostname. + Changing this value will trigger a rollout for all existing NodePools in the cluster. properties: additionalTrustedCA: description: |- @@ -1182,6 +1184,27 @@ spec: - v2 - "" type: string + minimumKubeletVersion: + description: |- + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 + type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible + format of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating @@ -1884,8 +1907,7 @@ spec: type: array type: object proxy: - description: Proxy holds cluster-wide information on how to configure - default proxies for the cluster. + description: ProxySpec contains cluster proxy creation configuration. properties: httpProxy: description: httpProxy is the URL of the proxy for HTTP requests. Empty @@ -2021,31 +2043,63 @@ spec: description: |- ControllerAvailabilityPolicy specifies the availability policy applied to critical control plane components. The default value is SingleReplica. + enum: + - HighlyAvailable + - SingleReplica type: string x-kubernetes-validations: - message: ControllerAvailabilityPolicy is immutable rule: self == oldSelf dns: - description: DNSSpec specifies the DNS configuration in the cluster. + description: DNSSpec specifies the DNS configuration for the hosted + cluster ingress. properties: baseDomain: - description: BaseDomain is the base domain of the cluster. + description: |- + baseDomain is the base domain of the hosted cluster. + It will be used to confgure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is ommitted, the hostedCluster.name will be used as the subdomain. + Once set, this field is immutable. + When the value is the empty string "", the controller might default to a value depending on the platform. + maxLength: 253 type: string + x-kubernetes-validations: + - message: baseDomain is immutable + rule: oldSelf == "" || self == oldSelf baseDomainPrefix: description: |- - BaseDomainPrefix is the base domain prefix of the cluster. - defaults to clusterName if not set. Set it to "" if you don't want a prefix to be prepended to BaseDomain. + baseDomainPrefix is the base domain prefix for the hosted cluster ingress. + It will be used to confgure ingress in the hosted cluster through the subdomain baseDomainPrefix.baseDomain. + If baseDomainPrefix is ommitted, the hostedCluster.name will be used as the subdomain. + Set baseDomainPrefix to an empty string "", if you don't want a prefix at all (not even hostedCluster.name) to be prepended to baseDomain. + This field is immutable. + maxLength: 253 type: string + x-kubernetes-validations: + - message: baseDomainPrefix is immutable + rule: self == oldSelf privateZoneID: description: |- - PrivateZoneID is the Hosted Zone ID where all the DNS records that are only - available internally to the cluster exist. + privateZoneID is the Hosted Zone ID where all the DNS records that are only available internally to the cluster exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: privateZoneID is immutable + rule: oldSelf == "" || self == oldSelf publicZoneID: description: |- - PublicZoneID is the Hosted Zone ID where all the DNS records that are - publicly accessible to the internet exist. + publicZoneID is the Hosted Zone ID where all the DNS records that are publicly accessible to the internet exist. + This field is optional and mainly leveraged in cloud environments where the DNS records for the .baseDomain are created by controllers in this zone. + Once set, this value is immutable. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: publicZoneID is immutable + rule: oldSelf == "" || self == oldSelf required: - baseDomain type: object @@ -2055,15 +2109,15 @@ spec: use to store data. properties: managed: - description: Managed specifies the behavior of an etcd cluster + description: managed specifies the behavior of an etcd cluster managed by HyperShift. properties: storage: - description: Storage specifies how etcd data is persisted. + description: storage specifies how etcd data is persisted. properties: persistentVolume: description: |- - PersistentVolume is the configuration for PersistentVolume etcd storage. + persistentVolume is the configuration for PersistentVolume etcd storage. With this implementation, a PersistentVolume will be allocated for every etcd member (either 1 or 3 depending on the HostedCluster control plane availability configuration). @@ -2073,8 +2127,10 @@ spec: - type: integer - type: string default: 8Gi - description: Size is the minimum size of the data - volume for each etcd member. + description: |- + size is the minimum size of the data volume for each etcd member. + Default is 8Gi. + This field is immutable pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true x-kubernetes-validations: @@ -2082,14 +2138,16 @@ spec: rule: self == oldSelf storageClassName: description: |- - StorageClassName is the StorageClass of the data volume for each etcd member. - + storageClassName is the StorageClass of the data volume for each etcd member. See https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1. type: string + x-kubernetes-validations: + - message: storageClassName is immutable + rule: self == oldSelf type: object restoreSnapshotURL: description: |- - RestoreSnapshotURL allows an optional URL to be provided where + restoreSnapshotURL allows an optional URL to be provided where an etcd snapshot can be downloaded, for example a pre-signed URL referencing a storage service. This snapshot will be restored on initial startup, only when the etcd PV @@ -2102,8 +2160,9 @@ spec: 1 entry rule: self.size() <= 1 type: - description: Type is the kind of persistent storage implementation - to use for etcd. + description: |- + type is the kind of persistent storage implementation to use for etcd. + Only PersistentVolume is supported at the moment. enum: - PersistentVolume type: string @@ -2114,19 +2173,25 @@ spec: - storage type: object managementType: - description: ManagementType defines how the etcd cluster is managed. + description: |- + managementType defines how the etcd cluster is managed. + This can be either Managed or Unmanaged. + This field is immutable. enum: - Managed - Unmanaged type: string + x-kubernetes-validations: + - message: managementType is immutable + rule: self == oldSelf unmanaged: description: |- - Unmanaged specifies configuration which enables the control plane to - integrate with an eternally managed etcd cluster. + unmanaged specifies configuration which enables the control plane to + integrate with an externally managed etcd cluster. properties: endpoint: description: |- - Endpoint is the full etcd cluster client endpoint URL. For example: + endpoint is the full etcd cluster client endpoint URL. For example: https://etcd-client:2379 @@ -2134,7 +2199,7 @@ spec: pattern: ^https:// type: string tls: - description: TLS specifies TLS configuration for HTTPS etcd + description: tls specifies TLS configuration for HTTPS etcd client endpoints. properties: clientSecret: @@ -2167,6 +2232,15 @@ spec: required: - managementType type: object + x-kubernetes-validations: + - message: Only managed configuration must be set when managementType + is Managed + rule: 'self.managementType == ''Managed'' ? has(self.managed) : + !has(self.managed)' + - message: Only unmanaged configuration must be set when managementType + is Unmanaged + rule: 'self.managementType == ''Unmanaged'' ? has(self.unmanaged) + : !has(self.unmanaged)' fips: description: FIPS specifies if the nodes for the cluster will be running in FIPS mode @@ -2204,6 +2278,9 @@ spec: InfrastructureAvailabilityPolicy specifies the availability policy applied to infrastructure services which run on cluster nodes. The default value is SingleReplica. + enum: + - HighlyAvailable + - SingleReplica type: string issuerURL: description: |- @@ -2244,19 +2321,23 @@ spec: properties: apiServer: description: |- - APIServer contains advanced network settings for the API server that affect - how the APIServer is exposed inside a cluster node. + apiServer contains advanced network settings for the API server that affect + how the APIServer is exposed inside a hosted cluster node. properties: advertiseAddress: description: |- - AdvertiseAddress is the address that nodes will use to talk to the API + advertiseAddress is the address that pods within the nodes will use to talk to the API server. This is an address associated with the loopback adapter of each node. If not specified, the controller will take default values. The default values will be set as 172.20.0.1 or fd00::1. + This value is immutable. type: string + x-kubernetes-validations: + - message: advertiseAddress is immutable + rule: self == oldSelf allowedCIDRBlocks: description: |- - AllowedCIDRBlocks is an allow list of CIDR blocks that can access the APIServer + allowedCIDRBlocks is an allow list of CIDR blocks that can access the APIServer If not specified, traffic is allowed from all addresses. This depends on underlying support by the cloud provider for Service LoadBalancerSourceRanges items: @@ -2265,42 +2346,64 @@ spec: type: array port: description: |- - Port is the port at which the APIServer is exposed inside a node. Other + port is the port at which the APIServer is exposed inside a node. Other pods using host networking cannot listen on this port. - If unset 6443 is used. + If ommited 6443 is used. This is useful to choose a port other than the default one which might interfere with customer environments e.g. https://github.com/openshift/hypershift/pull/356. Setting this to 443 is possible only for backward compatibility reasons and it's discouraged. Doing so, it would result in the controller overriding the KAS endpoint in the guest cluster having a discrepancy with the KAS Pod and potentially causing temporarily network failures. + This value is immutable. format: int32 type: integer + x-kubernetes-validations: + - message: port is immutable + rule: self == oldSelf type: object clusterNetwork: default: - cidr: 10.132.0.0/14 - description: ClusterNetwork is the list of IP address pools for - pods. + description: |- + clusterNetwork is the list of IP address pools for pods. + Defaults to cidr: "10.132.0.0/14". + Currently only one entry is supported. + This field is immutable. items: description: |- ClusterNetworkEntry is a single IP address block for pod IP blocks. IP blocks are allocated with size 2^HostSubnetLength. properties: cidr: - description: CIDR is the IP block address pool. + description: cidr is the IP block address pool. + maxLength: 43 type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') hostPrefix: description: |- - HostPrefix is the prefix size to allocate to each node from the CIDR. - For example, 24 would allocate 2^8=256 adresses to each node. If this + hostPrefix is the prefix size to allocate to each node from the CIDR. + For example, 24 would allocate 2^(32-24)=2^8=256 adresses to each node. If this field is not used by the plugin, it can be left unset. format: int32 type: integer required: - cidr type: object + maxItems: 2 + minItems: 1 type: array + x-kubernetes-validations: + - message: clusterNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf machineNetwork: - description: MachineNetwork is the list of IP address pools for - machines. + description: |- + machineNetwork is the list of IP address pools for machines. + This might be used among other things to generate appropriate networking security groups in some clouds providers. + Currently only one entry or two for dual stack is supported. + This field is immutable. items: description: MachineNetworkEntry is a single IP address block for node IP blocks. @@ -2308,15 +2411,30 @@ spec: cidr: description: CIDR is the IP block address pool for machines within the cluster. + maxLength: 43 type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') required: - cidr type: object + maxItems: 2 + minItems: 1 type: array + x-kubernetes-validations: + - message: machineNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf networkType: default: OVNKubernetes - description: NetworkType specifies the SDN provider used for cluster - networking. + description: |- + networkType specifies the SDN provider used for cluster networking. + Defaults to OVNKubernetes. + This field is required and immutable. + kubebuilder:validation:XValidation:rule="self == oldSelf", message="networkType is immutable" enum: - OpenShiftSDN - Calico @@ -2327,24 +2445,44 @@ spec: default: - cidr: 172.31.0.0/16 description: |- - ServiceNetwork is the list of IP address pools for services. - NOTE: currently only one entry is supported. + serviceNetwork is the list of IP address pools for services. + Defaults to cidr: "172.31.0.0/16". + Currently only one entry is supported. + This field is immutable. items: description: ServiceNetworkEntry is a single IP address block for the service network. properties: cidr: - description: CIDR is the IP block address pool for services - within the cluster. + description: cidr is the IP block address pool for services + within the cluster in CIDR format (e.g., 192.168.1.0/24 + or 2001:0db8::/64) + maxLength: 43 type: string + x-kubernetes-validations: + - message: cidr must be a valid IPv4 or IPv6 CIDR notation + (e.g., 192.168.1.0/24 or 2001:db8::/64) + rule: self.matches('^((\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2})$') + || self.matches('^([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?/[0-9]{1,3}$') required: - cidr type: object + maxItems: 2 + minItems: 1 type: array - required: - - clusterNetwork - - networkType + x-kubernetes-validations: + - message: serviceNetwork is immutable and cannot be modified + once set. + rule: self == oldSelf type: object + x-kubernetes-validations: + - message: CIDR ranges in machineNetwork, clusterNetwork, and serviceNetwork + must be unique and non-overlapping + rule: (!has(self.machineNetwork) && self.clusterNetwork.all(c, self.serviceNetwork.all(s, + c.cidr != s.cidr)) || (has(self.machineNetwork) && (self.machineNetwork.all(m, + self.clusterNetwork.all(c, m.cidr != c.cidr)) && self.machineNetwork.all(m, + self.serviceNetwork.all(s, m.cidr != s.cidr)) && self.clusterNetwork.all(c, + self.serviceNetwork.all(s, c.cidr != s.cidr))))) nodeSelector: additionalProperties: type: string @@ -2872,13 +3010,43 @@ spec: rule: self == oldSelf subnetID: description: |- - SubnetID is the subnet ID of an existing subnet where the load balancer for node egress will be created. This - subnet is expected to be a subnet within the VNET specified in VnetID. This subnet is expected to exist under the - same subscription as SubscriptionID. - - In ARO HCP, managed services will create the aforementioned load balancer in ResourceGroupName. + subnetID is the subnet ID of an existing subnet where the nodes in the nodepool will be created. This can be a + different subnet than the one listed in the HostedCluster, HostedCluster.Spec.Platform.Azure.SubnetID, but must + exist in the same network, HostedCluster.Spec.Platform.Azure.VnetID, and must exist under the same subscription ID, + HostedCluster.Spec.Platform.Azure.SubscriptionID. + subnetID is immutable once set. + The subnetID should be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}`. + The subscriptionId in the encryptionSetID must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + The resourceGroupName should be between 1 and 90 characters, consisting only of alphanumeric characters, hyphens, underscores, periods and paranthesis and must not end with a period (.) character. + The vnetName should be between 2 and 64 characters, consisting only of alphanumeric characters, hyphens, underscores and periods and must not end with either a period (.) or hyphen (-) character. + The subnetName should be between 1 and 80 characters, consisting only of alphanumeric characters, hyphens and underscores and must start with an alphanumeric character and must not end with a period (.) or hyphen (-) character. + maxLength: 355 + minLength: 1 type: string x-kubernetes-validations: + - message: encryptionSetID must be in the format `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}` + rule: size(self.split('/')) == 11 && self.matches('^/subscriptions/.*/resourceGroups/.*/providers/Microsoft.Network/virtualNetworks/.*/subnets/.*$') + - message: The resourceGroupName should be between 1 and 90 + characters, consisting only of alphanumeric characters, + hyphens, underscores, periods and paranthesis + rule: self.split('/')[4].matches('[a-zA-Z0-9-_\\(\\)\\.]{1,90}') + - message: the resourceGroupName in the subnetID must not + end with a period (.) character + rule: '!self.split(''/'')[4].endsWith(''.'')' + - message: The vnetName should be between 2 and 64 characters, + consisting only of alphanumeric characters, hyphens, underscores + and periods + rule: self.split('/')[8].matches('[a-zA-Z0-9-_\\.]{2,64}') + - message: the vnetName in the subnetID must not end with + either a period (.) or hyphen (-) character + rule: '!self.split(''/'')[8].endsWith(''.'') && !self.split(''/'')[8].endsWith(''-'')' + - message: The subnetName should be between 1 and 80 characters, + consisting only of alphanumeric characters, hyphens and + underscores and must start with an alphanumeric character + rule: self.split('/')[10].matches('[a-zA-Z0-9][a-zA-Z0-9-_\\.]{0,79}') + - message: the subnetName in the subnetID must not end with + a period (.) or hyphen (-) character + rule: '!self.split(''/'')[10].endsWith(''.'') && !self.split(''/'')[10].endsWith(''-'')' - message: SubnetID is immutable rule: self == oldSelf subscriptionID: @@ -3601,11 +3769,16 @@ spec: in the management cluster. items: description: |- - ServicePublishingStrategyMapping specifies how individual control plane - services are published from the hosting cluster of a control plane. + ServicePublishingStrategyMapping specifies how individual control plane services endpoints are published for consumption. + This includes APIServer;OAuthServer;Konnectivity;Ignition. + If a given service is not present in this list, it will be exposed publicly by default. properties: service: - description: Service identifies the type of service being published. + description: |- + service identifies the type of service being published. + It can be APIServer;OAuthServer;Konnectivity;Ignition + OVNSbDb;OIDC are no-op and kept for backward compatibility. + This field is immutable. enum: - APIServer - OAuthServer @@ -3615,29 +3788,44 @@ spec: - OVNSbDb type: string servicePublishingStrategy: - description: ServicePublishingStrategy specifies how to publish - Service. + description: servicePublishingStrategy specifies how to publish + a service endpoint. properties: loadBalancer: - description: LoadBalancer configures exposing a service - using a LoadBalancer. + description: loadBalancer configures exposing a service + using a dedicated LoadBalancer. properties: hostname: - description: Hostname is the name of the DNS record - that will be created pointing to the LoadBalancer. + description: |- + hostname is the name of the DNS record that will be created pointing to the LoadBalancer and passed through to consumers of the service. + If ommited, the value will be infered from the corev1.Service Load balancer type .status. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: baseDomain must be a valid base domain (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') type: object nodePort: - description: NodePort configures exposing a service using + description: nodePort configures exposing a service using a NodePort. properties: address: - description: Address is the host/ip that the NodePort + description: address is the host/ip that the NodePort service is exposed over. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: address must be a valid hostname, IPv4, or + IPv6 address + rule: self.matches('^(([a-zA-Z0-9][-a-zA-Z0-9]*\\.)+[a-zA-Z]{2,}|localhost)$') + || self.matches('^((\\d{1,3}\\.){3}\\d{1,3})$') + || self.matches('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$') port: description: |- - Port is the port of the NodePort service. If <=0, the port is dynamically + port is the port of the NodePort service. If <=0, the port is dynamically assigned when the service is created. format: int32 type: integer @@ -3645,17 +3833,26 @@ spec: - address type: object route: - description: Route configures exposing a service using a - Route. + description: |- + route configures exposing a service using a Route through and an ingress controller behind a cloud Load Balancer. + The specifics of the setup are platform dependent. properties: hostname: - description: Hostname is the name of the DNS record - that will be created pointing to the Route. + description: |- + Hostname is the name of the DNS record that will be created pointing to the Route and passed through to consumers of the service. + If ommited, the value will be infered from management ingress.Spec.Domain. + maxLength: 253 + minLength: 1 type: string + x-kubernetes-validations: + - message: hostname must be a valid domain name (e.g., + example.com) + rule: self.matches('^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}$') type: object type: - description: Type is the publishing strategy used for the - service. + description: |- + type is the publishing strategy used for the service. + It can be LoadBalancer;NodePort;Route;None;S3 enum: - LoadBalancer - NodePort @@ -3666,10 +3863,32 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: nodePort is required when type is NodePort, and forbidden + otherwise + rule: 'self.type == ''NodePort'' ? has(self.nodePort) : !has(self.nodePort)' + - message: only route is allowed when type is Route, and forbidden + otherwise + rule: 'self.type == ''Route'' ? !has(self.nodePort) && !has(self.loadBalancer) + : !has(self.route)' + - message: only loadBalancer is required when type is LoadBalancer, + and forbidden otherwise + rule: 'self.type == ''LoadBalancer'' ? !has(self.nodePort) + && !has(self.route) : !has(self.loadBalancer)' + - message: None does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''None'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' + - message: S3 does not allowed any configuration for loadBalancer, + nodePort, or route + rule: 'self.type == ''S3'' ? !has(self.nodePort) && !has(self.route) + && !has(self.loadBalancer) : true' required: - service - servicePublishingStrategy type: object + maxItems: 6 + minItems: 4 type: array sshKey: description: |- @@ -3888,6 +4107,10 @@ spec: with the name of an identity provider defined on the HostedCluster. This is populated after the infrastructure is ready. type: string + oldestKubeletVersion: + description: OldestKubeletVersion tracks the oldest kubelet version + in a hosted cluster + type: string platform: description: Platform contains platform-specific status of the HostedCluster properties: diff --git a/client/applyconfiguration/hypershift/v1beta1/clusterconfiguration.go b/client/applyconfiguration/hypershift/v1beta1/clusterconfiguration.go index 8f27725e4d..c089b8b1bd 100644 --- a/client/applyconfiguration/hypershift/v1beta1/clusterconfiguration.go +++ b/client/applyconfiguration/hypershift/v1beta1/clusterconfiguration.go @@ -34,6 +34,7 @@ type ClusterConfigurationApplyConfiguration struct { OperatorHub *v1.OperatorHubSpec `json:"operatorhub,omitempty"` Scheduler *v1.SchedulerSpec `json:"scheduler,omitempty"` Proxy *v1.ProxySpec `json:"proxy,omitempty"` + Node *v1.NodeSpec `json:"node,omitempty"` } // ClusterConfigurationApplyConfiguration constructs an declarative configuration of the ClusterConfiguration type for use with @@ -121,3 +122,11 @@ func (b *ClusterConfigurationApplyConfiguration) WithProxy(value v1.ProxySpec) * b.Proxy = &value return b } + +// WithNode sets the Node field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Node field is set to the value of the last call. +func (b *ClusterConfigurationApplyConfiguration) WithNode(value v1.NodeSpec) *ClusterConfigurationApplyConfiguration { + b.Node = &value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/hostedclusterstatus.go b/client/applyconfiguration/hypershift/v1beta1/hostedclusterstatus.go index 433804746c..303120b38e 100644 --- a/client/applyconfiguration/hypershift/v1beta1/hostedclusterstatus.go +++ b/client/applyconfiguration/hypershift/v1beta1/hostedclusterstatus.go @@ -35,6 +35,7 @@ type HostedClusterStatusApplyConfiguration struct { Conditions []metav1.ConditionApplyConfiguration `json:"conditions,omitempty"` PayloadArch *hypershiftv1beta1.PayloadArchType `json:"payloadArch,omitempty"` Platform *PlatformStatusApplyConfiguration `json:"platform,omitempty"` + OldestKubeletVersion *string `json:"oldestKubeletVersion,omitempty"` } // HostedClusterStatusApplyConfiguration constructs an declarative configuration of the HostedClusterStatus type for use with @@ -119,3 +120,11 @@ func (b *HostedClusterStatusApplyConfiguration) WithPlatform(value *PlatformStat b.Platform = value return b } + +// WithOldestKubeletVersion sets the OldestKubeletVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the OldestKubeletVersion field is set to the value of the last call. +func (b *HostedClusterStatusApplyConfiguration) WithOldestKubeletVersion(value string) *HostedClusterStatusApplyConfiguration { + b.OldestKubeletVersion = &value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanestatus.go b/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanestatus.go index bd78944b3c..af0de3ac78 100644 --- a/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanestatus.go +++ b/client/applyconfiguration/hypershift/v1beta1/hostedcontrolplanestatus.go @@ -40,6 +40,7 @@ type HostedControlPlaneStatusApplyConfiguration struct { Conditions []metav1.ConditionApplyConfiguration `json:"conditions,omitempty"` Platform *PlatformStatusApplyConfiguration `json:"platform,omitempty"` NodeCount *int `json:"nodeCount,omitempty"` + OldestKubeletVersion *string `json:"oldestKubeletVersion,omitempty"` } // HostedControlPlaneStatusApplyConfiguration constructs an declarative configuration of the HostedControlPlaneStatus type for use with @@ -164,3 +165,11 @@ func (b *HostedControlPlaneStatusApplyConfiguration) WithNodeCount(value int) *H b.NodeCount = &value return b } + +// WithOldestKubeletVersion sets the OldestKubeletVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the OldestKubeletVersion field is set to the value of the last call. +func (b *HostedControlPlaneStatusApplyConfiguration) WithOldestKubeletVersion(value string) *HostedControlPlaneStatusApplyConfiguration { + b.OldestKubeletVersion = &value + return b +} diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml index 13fd76d07c..a567673638 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml @@ -1642,6 +1642,27 @@ spec: - v2 - "" type: string + minimumKubeletVersion: + description: |- + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 + type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible + format of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating @@ -5421,6 +5442,10 @@ spec: with the name of an identity provider defined on the HostedCluster. This is populated after the infrastructure is ready. type: string + oldestKubeletVersion: + description: OldestKubeletVersion tracks the oldest kubelet version + in a hosted cluster + type: string payloadArch: description: |- payloadArch represents the CPU architecture type of the HostedCluster.Spec.Release.Image. The valid values are: diff --git a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml index 98ba15b3bd..2b8016d7cf 100644 --- a/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml @@ -1599,6 +1599,27 @@ spec: - v2 - "" type: string + minimumKubeletVersion: + description: |- + minimumKubeletVersion is the lowest version of a kubelet that can join the cluster. + Specifically, the apiserver will deny most authorization requests of kubelets that are older + than the specified version, only allowing the kubelet to get and update its node object, and perform + subjectaccessreviews. + This means any kubelet that attempts to join the cluster will not be able to run any assigned workloads, + and will eventually be marked as not ready. + Its max length is 8, so maximum version allowed is either "9.999.99" or "99.99.99". + Since the kubelet reports the version of the kubernetes release, not Openshift, this field references + the underlying kubernetes version this version of Openshift is based off of. + In other words: if an admin wishes to ensure no nodes run an older version than Openshift 4.17, then + they should set the minimumKubeletVersion to 1.30.0. + When comparing versions, the kubelet's version is stripped of any contents outside of major.minor.patch version. + Thus, a kubelet with version "1.0.0-ec.0" will be compatible with minimumKubeletVersion "1.0.0" or earlier. + maxLength: 8 + type: string + x-kubernetes-validations: + - message: minmumKubeletVersion must be in a semver compatible + format of x.y.z, or empty + rule: self == "" || self.matches('^[0-9]*.[0-9]*.[0-9]*$') workerLatencyProfile: description: |- WorkerLatencyProfile determins the how fast the kubelet is updating @@ -5265,6 +5286,10 @@ spec: with the name of an identity provider defined on the HostedCluster. This is populated after the infrastructure is ready. type: string + oldestKubeletVersion: + description: OldestKubeletVersion tracks the oldest kubelet version + in a hosted cluster + type: string platform: description: Platform contains platform-specific status of the HostedCluster properties: diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index c3d7a567ef..65fc366a4e 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -5479,6 +5479,17 @@ PlatformStatus

Platform contains platform-specific status of the HostedCluster

+ + +oldestKubeletVersion
+ +string + + + +

OldestKubeletVersion tracks the oldest kubelet version in a hosted cluster

+ + ###HostedControlPlaneSpec { #hypershift.openshift.io/v1beta1.HostedControlPlaneSpec } @@ -6126,6 +6137,17 @@ int

NodeCount tracks the number of nodes in the HostedControlPlane.

+ + +oldestKubeletVersion
+ +string + + + +

OldestKubeletVersion tracks the oldest kubelet version in a hosted cluster

+ + ###IBMCloudKMSAuthSpec { #hypershift.openshift.io/v1beta1.IBMCloudKMSAuthSpec } diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go index a9aa8e81a9..b5825bc7c7 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hosted_controlplane.go @@ -318,6 +318,10 @@ type HostedControlPlaneStatus struct { // NodeCount tracks the number of nodes in the HostedControlPlane. NodeCount *int `json:"nodeCount,omitempty"` + + // OldestKubeletVersion tracks the oldest kubelet version in a hosted cluster + // +openshift:enable:FeatureGate=MinimumKubeletVersion + OldestKubeletVersion *string `json:"oldestKubeletVersion,omitempty"` } type APIEndpoint struct { diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go index 209efa02ca..ae224025cf 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go @@ -1414,6 +1414,10 @@ type HostedClusterStatus struct { // Platform contains platform-specific status of the HostedCluster // +optional Platform *PlatformStatus `json:"platform,omitempty"` + + // OldestKubeletVersion tracks the oldest kubelet version in a hosted cluster + // +openshift:enable:FeatureGate=MinimumKubeletVersion + OldestKubeletVersion *string `json:"oldestKubeletVersion,omitempty"` } // PlatformStatus contains platform-specific status diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index cd06a4e580..935f9f2880 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -1443,6 +1443,11 @@ func (in *HostedClusterStatus) DeepCopyInto(out *HostedClusterStatus) { *out = new(PlatformStatus) (*in).DeepCopyInto(*out) } + if in.OldestKubeletVersion != nil { + in, out := &in.OldestKubeletVersion, &out.OldestKubeletVersion + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostedClusterStatus. @@ -1656,6 +1661,11 @@ func (in *HostedControlPlaneStatus) DeepCopyInto(out *HostedControlPlaneStatus) *out = new(int) **out = **in } + if in.OldestKubeletVersion != nil { + in, out := &in.OldestKubeletVersion, &out.OldestKubeletVersion + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostedControlPlaneStatus.