diff --git a/api/operator/v1beta1/vmauth_types_test.go b/api/operator/v1beta1/vmauth_types_test.go index 06bafaf33..79d901d04 100644 --- a/api/operator/v1beta1/vmauth_types_test.go +++ b/api/operator/v1beta1/vmauth_types_test.go @@ -26,13 +26,13 @@ func TestVMAuthValidate(t *testing.T) { // invalid ingress f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail spec: ingress: - tlsHosts: + tlsHosts: - host-1 - host-2`, wantErr: `spec.ingress.tlsSecretName cannot be empty with non-empty spec.ingress.tlsHosts`, @@ -41,7 +41,7 @@ spec: // both configSecret and external config is defined at the same time f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail @@ -57,13 +57,13 @@ spec: // incorrect unauthorized access config, missing backends" f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail spec: unauthorizedUserAccessSpec: - default_url: + default_url: - http://url-1`, wantErr: "incorrect cr.spec.UnauthorizedUserAccess syntax: at least one of `url_map`, `url_prefix` or `targetRefs` must be defined", }) @@ -71,7 +71,7 @@ spec: // incorrect unauthorized access config, bad metric_labels syntax f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail @@ -80,7 +80,7 @@ spec: metric_labels: 124124asff: 12fsaf url_prefix: http://some-dst - default_url: + default_url: - http://url-1`, wantErr: `incorrect cr.spec.UnauthorizedUserAccess syntax: incorrect metricLabelName="124124asff", must match pattern="^[a-zA-Z_:.][a-zA-Z0-9_:.]*$"`, }) @@ -88,7 +88,7 @@ spec: // incorrect unauthorized access config url_map" f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail @@ -100,7 +100,7 @@ spec: - url_prefix: http://some-url src_paths: ["/path-1"] - url_prefix: http://some-url-2 - default_url: + default_url: - http://url-1`, wantErr: `incorrect cr.spec.UnauthorizedUserAccess syntax: incorrect url_map at idx=1: incorrect url_map config at least of one src_paths,src_hosts,src_query_args or src_headers must be defined`, }) @@ -108,12 +108,12 @@ spec: // both unauthorizedUserAccessSpec and UnauthorizedUserAccess defined f(opts{ src: ` -apiVersion: v1 +apiVersion: v1 kind: VMAuth metadata: name: must-fail spec: - unauthorizedAccessConfig: + unauthorizedAccessConfig: - url_prefix: http://some-url src_paths: ["/path-1"] - url_prefix: http://some-url-2 @@ -124,7 +124,7 @@ spec: url_map: - url_prefix: http://some-url src_paths: ["/path-1"] - default_url: + default_url: - http://url-1`, wantErr: "at most one option can be used `spec.unauthorizedAccessConfig` or `spec.unauthorizedUserAccessSpec`, got both", }) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2125d814e..781dc4378 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -27,6 +27,8 @@ aliases: * FEATURE: [vmdistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed): introduce `spec.zones[*].trafficMode` property, which allows disable read, write or whole traffic to a zone. See [#1995](https://github.com/VictoriaMetrics/operator/issues/1995). * FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): support per remote write queues configuration. See [#2138](https://github.com/VictoriaMetrics/operator/issues/2138). +* FEATURE: [vmoperator](https://docs.victoriametrics.com/operator/): added `VM_COMMON_LABELS` and `VM_COMMON_ANNOTATIONS` environment variables to apply common labels/annotations to all Kubernetes resources managed by the operator. These cannot override labels/annotations already set by the operator or via `spec.managedMetadata`. This also ensures HTTPRoutes and PVCs include ManagedMetadata labels and annotations + * BUGFIX: [converter](https://docs.victoriametrics.com/operator/integrations/prometheus/#objects-conversion): disable all prometheus controllers if CRD group was not found. See [#2838](https://github.com/VictoriaMetrics/helm-charts/issues/2838). * BUGFIX: [vmdistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/): change default load balancing policy for write requests from `first_available` to `least_loaded`. This should allow to evenly distribute write load across all VMAgents. * BUGFIX: [vmalertmanagerconfig](https://docs.victoriametrics.com/operator/resources/vmalertmanagerconfig/): fix previously ignored negative values in VMAlertmanagerConfig. See [#2132](https://github.com/VictoriaMetrics/operator/issues/2132). diff --git a/docs/env.md b/docs/env.md index 1d0d81084..31d053cca 100644 --- a/docs/env.md +++ b/docs/env.md @@ -4,7 +4,7 @@ | VM_LOGS_VERSION: `v1.50.0` # | | VM_ANOMALY_VERSION: `v1.29.3` # | | VM_TRACES_VERSION: `v0.7.0` # | -| VM_OPERATOR_VERSION: `v0.70.0` # | +| VM_OPERATOR_VERSION: `v0.69.0` # | | VM_GATEWAY_API_ENABLED: `false` # | | VM_VPA_API_ENABLED: `false` # | | WATCH_NAMESPACE: `-` #
Defines a list of namespaces to be watched by operator. Operator don't perform any cluster wide API calls if namespaces not empty. In case of empty list it performs only clusterwide api calls. | @@ -267,3 +267,5 @@ | VM_WAIT_READY_INTERVAL: `5s` #
Defines poll interval for VM CRs | | VM_FORCERESYNCINTERVAL: `60s` #
configures force resync interval for VMAgent, VMAlert, VMAlertmanager and VMAuth. | | VM_ENABLESTRICTSECURITY: `false` #
EnableStrictSecurity will add default `securityContext` to pods and containers created by operator Default PodSecurityContext include: 1. RunAsNonRoot: true 2. RunAsUser/RunAsGroup/FSGroup: 65534 '65534' refers to 'nobody' in all the used default images like alpine, busybox. If you're using customize image, please make sure '65534' is a valid uid in there or specify SecurityContext. 3. FSGroupChangePolicy: &onRootMismatch If KubeVersion>=1.20, use `FSGroupChangePolicy="onRootMismatch"` to skip the recursive permission change when the root of the volume already has the correct permissions 4. SeccompProfile: type: RuntimeDefault Use `RuntimeDefault` seccomp profile by default, which is defined by the container runtime, instead of using the Unconfined (seccomp disabled) mode. Default container SecurityContext include: 1. AllowPrivilegeEscalation: false 2. ReadOnlyRootFilesystem: true 3. Capabilities: drop: - all turn off `EnableStrictSecurity` by default, see https://github.com/VictoriaMetrics/operator/issues/749 for details | +| VM_COMMON_LABELS: `-` #
CommonLabels are added to every Kubernetes resource created by the operator. They cannot override labels already set by the operator or via spec.managedMetadata. Format: key=value,key2=value2 | +| VM_COMMON_ANNOTATIONS: `-` #
CommonAnnotations are added to every Kubernetes resource created by the operator. They cannot override annotations already set by the operator or via spec.managedMetadata. Format: key=value,key2=value2 | diff --git a/internal/config/config.go b/internal/config/config.go index f6a308d00..ebc54d17f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -623,6 +623,15 @@ type BaseOperatorConf struct { // - all // turn off `EnableStrictSecurity` by default, see https://github.com/VictoriaMetrics/operator/issues/749 for details EnableStrictSecurity bool `default:"false" env:"VM_ENABLESTRICTSECURITY"` + + // CommonLabels are added to every Kubernetes resource created by the operator. + // They cannot override labels already set by the operator or via spec.managedMetadata. + // Format: key=value,key2=value2 + CommonLabels map[string]string `default:"" env:"VM_COMMON_LABELS" envSeparator:"," envKeyValSeparator:"="` + // CommonAnnotations are added to every Kubernetes resource created by the operator. + // They cannot override annotations already set by the operator or via spec.managedMetadata. + // Format: key=value,key2=value2 + CommonAnnotations map[string]string `default:"" env:"VM_COMMON_ANNOTATIONS" envSeparator:"," envKeyValSeparator:"="` } // ResyncAfterDuration returns requeue duration for object period reconcile diff --git a/internal/controller/operator/factory/build/daemonset.go b/internal/controller/operator/factory/build/daemonset.go index df1d0d425..d5b282a20 100644 --- a/internal/controller/operator/factory/build/daemonset.go +++ b/internal/controller/operator/factory/build/daemonset.go @@ -2,34 +2,14 @@ package build import ( appsv1 "k8s.io/api/apps/v1" - "k8s.io/utils/ptr" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" ) -// DeploymentAddCommonParams adds common params for all deployments +// DaemonSetAddCommonParams adds common params for all deployments func DaemonSetAddCommonParams(dst *appsv1.DaemonSet, params *vmv1beta1.CommonAppsParams) { - dst.Spec.Template.Spec.Affinity = params.Affinity - dst.Spec.Template.Spec.Tolerations = params.Tolerations - dst.Spec.Template.Spec.SchedulerName = params.SchedulerName - dst.Spec.Template.Spec.RuntimeClassName = params.RuntimeClassName - dst.Spec.Template.Spec.HostAliases = params.HostAliases - if len(params.HostAliasesUnderScore) > 0 { - dst.Spec.Template.Spec.HostAliases = params.HostAliasesUnderScore - } - dst.Spec.Template.Spec.PriorityClassName = params.PriorityClassName - dst.Spec.Template.Spec.HostNetwork = params.HostNetwork - dst.Spec.Template.Spec.DNSPolicy = params.DNSPolicy - dst.Spec.Template.Spec.DNSConfig = params.DNSConfig - dst.Spec.Template.Spec.NodeSelector = params.NodeSelector + PodTemplateAddCommonParams(&dst.Spec.Template, params) dst.Spec.Template.Spec.SecurityContext = addStrictSecuritySettingsWithRootToPod(params) - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds - dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints - dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount - if params.DisableAutomountServiceAccountToken { - dst.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) - } } diff --git a/internal/controller/operator/factory/build/defaults.go b/internal/controller/operator/factory/build/defaults.go index b294ce7d3..5e93d70b0 100644 --- a/internal/controller/operator/factory/build/defaults.go +++ b/internal/controller/operator/factory/build/defaults.go @@ -7,6 +7,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" @@ -48,10 +49,19 @@ func AddDefaults(scheme *runtime.Scheme) { scheme.AddTypeDefaultingFunc(&vmv1.VMAnomaly{}, addVMAnomalyDefaults) scheme.AddTypeDefaultingFunc(&vmv1beta1.VMServiceScrape{}, addVMServiceScrapeDefaults) scheme.AddTypeDefaultingFunc(&vmv1alpha1.VMDistributed{}, addVMDistributedDefaults) + + scheme.AddTypeDefaultingFunc(&appsv1.DaemonSet{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.ConfigMap{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.Namespace{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.PersistentVolumeClaim{}, addDefaultMetadata) + scheme.AddTypeDefaultingFunc(&corev1.Secret{}, addDefaultMetadata) } func addVMDistributedDefaults(objI any) { cr := objI.(*vmv1alpha1.VMDistributed) + + addDefaultMetadata(cr) + if cr.Spec.ZoneCommon.ReadyTimeout == nil { cr.Spec.ZoneCommon.ReadyTimeout = &metav1.Duration{ Duration: 5 * time.Minute, @@ -86,6 +96,8 @@ func addVMDistributedDefaults(objI any) { func addStatefulsetDefaults(objI any) { obj := objI.(*appsv1.StatefulSet) + addDefaultMetadata(obj) + // special case for vm operator defaults if obj.Spec.UpdateStrategy.Type == "" { obj.Spec.UpdateStrategy.Type = appsv1.OnDeleteStatefulSetStrategyType @@ -123,6 +135,9 @@ func addStatefulsetDefaults(objI any) { // https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/apps/v1/defaults.go func addDeploymentDefaults(objI any) { obj := objI.(*appsv1.Deployment) + + addDefaultMetadata(obj) + // Set DeploymentSpec.Replicas to 1 if it is not set. if obj.Spec.Replicas == nil { obj.Spec.Replicas = new(int32) @@ -163,6 +178,9 @@ func addDeploymentDefaults(objI any) { // https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/core/v1/defaults.go func addServiceDefaults(objI any) { obj := objI.(*corev1.Service) + + addDefaultMetadata(obj) + if obj.Spec.SessionAffinity == "" { obj.Spec.SessionAffinity = corev1.ServiceAffinityNone } @@ -221,6 +239,8 @@ func addVMAuthDefaults(objI any) { cr := objI.(*vmv1beta1.VMAuth) c := getCfg() + addDefaultMetadata(cr) + if cr.Spec.ConfigSecret != "" { // Removed if later with ConfigSecret field later cr.Spec.SecretRef = &corev1.SecretKeySelector{ @@ -243,6 +263,8 @@ func addVMAlertDefaults(objI any) { cr := objI.(*vmv1beta1.VMAlert) c := getCfg() + addDefaultMetadata(cr) + cv := config.ApplicationDefaults(c.VMAlert) cp := commonParams{ tag: cr.Spec.ComponentVersion, @@ -258,6 +280,7 @@ func addVMAlertDefaults(objI any) { func addVMAgentDefaults(objI any) { cr := objI.(*vmv1beta1.VMAgent) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VMAgent) cp := commonParams{ @@ -274,6 +297,7 @@ func addVMAgentDefaults(objI any) { func addVLAgentDefaults(objI any) { cr := objI.(*vmv1.VLAgent) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VLAgent) cp := commonParams{ @@ -286,6 +310,7 @@ func addVLAgentDefaults(objI any) { func addVMSingleDefaults(objI any) { cr := objI.(*vmv1beta1.VMSingle) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VMSingle) cp := commonParams{ tag: cr.Spec.ComponentVersion, @@ -307,6 +332,7 @@ func addVMSingleDefaults(objI any) { func addVLogsDefaults(objI any) { cr := objI.(*vmv1beta1.VLogs) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VLogs) cp := commonParams{tag: cr.Spec.ComponentVersion} addDefaultsToCommonParams(&cr.Spec.CommonAppsParams, &cp, &cv) @@ -315,6 +341,8 @@ func addVLogsDefaults(objI any) { func addVMAnomalyDefaults(objI any) { cr := objI.(*vmv1.VMAnomaly) + addDefaultMetadata(cr) + // vmanomaly takes up to 2 minutes to start if cr.Spec.LivenessProbe == nil { cr.Spec.LivenessProbe = &corev1.Probe{ @@ -357,6 +385,7 @@ func addVMAnomalyDefaults(objI any) { func addVLSingleDefaults(objI any) { cr := objI.(*vmv1.VLSingle) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VLSingle) cp := commonParams{ tag: cr.Spec.ComponentVersion, @@ -368,6 +397,7 @@ func addVLSingleDefaults(objI any) { func addVTSingleDefaults(objI any) { cr := objI.(*vmv1.VTSingle) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VTSingle) cp := commonParams{tag: cr.Spec.ComponentVersion} addDefaultsToCommonParams(&cr.Spec.CommonAppsParams, &cp, &cv) @@ -376,6 +406,7 @@ func addVTSingleDefaults(objI any) { func addVMAlertmanagerDefaults(objI any) { cr := objI.(*vmv1beta1.VMAlertmanager) c := getCfg() + addDefaultMetadata(cr) cv := config.ApplicationDefaults(c.VMAlertmanager) if cr.Spec.ClusterDomainName == "" { cr.Spec.ClusterDomainName = c.ClusterDomainName @@ -414,6 +445,7 @@ func addRequestsLoadBalancerDefaults(lb *vmv1beta1.VMAuthLoadBalancer, cp *commo func addVMClusterDefaults(objI any) { cr := objI.(*vmv1beta1.VMCluster) c := getCfg() + addDefaultMetadata(cr) if cr.Spec.ClusterDomainName == "" { cr.Spec.ClusterDomainName = c.ClusterDomainName } @@ -467,6 +499,20 @@ func addVMClusterDefaults(objI any) { } } +func addDefaultMetadata(objI any) { + cfg := config.MustGetBaseConfig() + obj, ok := objI.(metav1.Object) + if !ok { + return + } + if len(cfg.CommonLabels) > 0 { + obj.SetLabels(labels.Merge(cfg.CommonLabels, obj.GetLabels())) + } + if len(cfg.CommonAnnotations) > 0 { + obj.SetAnnotations(labels.Merge(cfg.CommonAnnotations, obj.GetAnnotations())) + } +} + func addDefaultsToCommonParams(common *vmv1beta1.CommonAppsParams, cp *commonParams, appDefaults *config.ApplicationDefaults) { c := getCfg() if common.Image.Repository == "" { @@ -537,6 +583,7 @@ func addDefaultsToVMBackup(cr *vmv1beta1.VMBackup, useDefaultResources bool, app return } c := getCfg() + addDefaultMetadata(cr) if cr.Image.Repository == "" { cr.Image.Repository = appDefaults.Image @@ -562,6 +609,7 @@ func addVMServiceScrapeDefaults(objI any) { if cr == nil { return } + addDefaultMetadata(cr) c := getCfg() if cr.Spec.DiscoveryRole == "" && c.VMServiceScrape.EnforceEndpointSlices { cr.Spec.DiscoveryRole = "endpointslice" @@ -576,6 +624,7 @@ const ( func addVTClusterDefaults(objI any) { cr := objI.(*vmv1.VTCluster) c := getCfg() + addDefaultMetadata(cr) cp := commonParams{ useStrictSecurity: cr.Spec.UseStrictSecurity, tag: cr.Spec.ClusterVersion, @@ -619,6 +668,7 @@ func addVTClusterDefaults(objI any) { func addVLClusterDefaults(objI any) { cr := objI.(*vmv1.VLCluster) c := getCfg() + addDefaultMetadata(cr) cp := commonParams{ useStrictSecurity: cr.Spec.UseStrictSecurity, tag: cr.Spec.ClusterVersion, diff --git a/internal/controller/operator/factory/build/defaults_test.go b/internal/controller/operator/factory/build/defaults_test.go index 86203d29b..cea3727a6 100644 --- a/internal/controller/operator/factory/build/defaults_test.go +++ b/internal/controller/operator/factory/build/defaults_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" @@ -549,3 +551,42 @@ func TestClusterComponentVersionDefaults(t *testing.T) { }) } } + +func TestAddDefaultMetadata(t *testing.T) { + cfg := config.MustGetBaseConfig() + defaultCfg := *cfg + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + + cfg.CommonLabels = map[string]string{ + "common-label": "common-value", + "existing-label": "should-not-overwrite", + } + cfg.CommonAnnotations = map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "should-not-overwrite", + } + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "existing-label": "existing-value", + }, + Annotations: map[string]string{ + "existing-annotation": "existing-value", + }, + }, + } + + addDefaultMetadata(obj) + + assert.Equal(t, map[string]string{ + "common-label": "common-value", + "existing-label": "existing-value", + }, obj.Labels) + assert.Equal(t, map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "existing-value", + }, obj.Annotations) +} diff --git a/internal/controller/operator/factory/build/deployment.go b/internal/controller/operator/factory/build/deployment.go index 5b38cc2ad..5a7e0de9c 100644 --- a/internal/controller/operator/factory/build/deployment.go +++ b/internal/controller/operator/factory/build/deployment.go @@ -2,35 +2,14 @@ package build import ( appsv1 "k8s.io/api/apps/v1" - "k8s.io/utils/ptr" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" ) // DeploymentAddCommonParams adds common params for all deployments func DeploymentAddCommonParams(dst *appsv1.Deployment, params *vmv1beta1.CommonAppsParams) { - dst.Spec.Template.Spec.Affinity = params.Affinity - dst.Spec.Template.Spec.Tolerations = params.Tolerations - dst.Spec.Template.Spec.SchedulerName = params.SchedulerName - dst.Spec.Template.Spec.RuntimeClassName = params.RuntimeClassName - dst.Spec.Template.Spec.HostAliases = params.HostAliases - if len(params.HostAliasesUnderScore) > 0 { - dst.Spec.Template.Spec.HostAliases = params.HostAliasesUnderScore - } - dst.Spec.Template.Spec.PriorityClassName = params.PriorityClassName - dst.Spec.Template.Spec.HostNetwork = params.HostNetwork - dst.Spec.Template.Spec.DNSPolicy = params.DNSPolicy - dst.Spec.Template.Spec.DNSConfig = params.DNSConfig - dst.Spec.Template.Spec.NodeSelector = params.NodeSelector - dst.Spec.Template.Spec.SecurityContext = addStrictSecuritySettingsToPod(params) - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds - dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints - dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates + PodTemplateAddCommonParams(&dst.Spec.Template, params) dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.Replicas = params.ReplicaCount dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount - if params.DisableAutomountServiceAccountToken { - dst.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) - } } diff --git a/internal/controller/operator/factory/build/httproute.go b/internal/controller/operator/factory/build/httproute.go index a5143e849..3f53f595c 100644 --- a/internal/controller/operator/factory/build/httproute.go +++ b/internal/controller/operator/factory/build/httproute.go @@ -16,7 +16,7 @@ import ( // HTTPRoute creates HTTPRoute object func HTTPRoute(cr builderOpts, port string, httpRoute *vmv1beta1.EmbeddedHTTPRoute) (*gwapiv1.HTTPRoute, error) { - lbls := labels.Merge(httpRoute.Labels, cr.SelectorLabels()) + lbls := labels.Merge(labels.Merge(cr.FinalLabels(), httpRoute.Labels), cr.SelectorLabels()) spec := gwapiv1.HTTPRouteSpec{ CommonRouteSpec: gwapiv1.CommonRouteSpec{ @@ -35,7 +35,7 @@ func HTTPRoute(cr builderOpts, port string, httpRoute *vmv1beta1.EmbeddedHTTPRou Name: cr.PrefixedName(), Namespace: cr.GetNamespace(), Labels: lbls, - Annotations: httpRoute.Annotations, + Annotations: labels.Merge(cr.FinalAnnotations(), httpRoute.Annotations), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: spec, diff --git a/internal/controller/operator/factory/build/podtemplate.go b/internal/controller/operator/factory/build/podtemplate.go new file mode 100644 index 000000000..9a6f7065d --- /dev/null +++ b/internal/controller/operator/factory/build/podtemplate.go @@ -0,0 +1,43 @@ +package build + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" + + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" +) + +// PodTemplateAddCommonParams populates common pod-level fields on a PodTemplateSpec +func PodTemplateAddCommonParams(dst *corev1.PodTemplateSpec, params *vmv1beta1.CommonAppsParams) { + dst.Spec.Affinity = params.Affinity + dst.Spec.Tolerations = params.Tolerations + dst.Spec.SchedulerName = params.SchedulerName + dst.Spec.RuntimeClassName = params.RuntimeClassName + dst.Spec.HostAliases = params.HostAliases + if len(params.HostAliasesUnderScore) > 0 { + dst.Spec.HostAliases = params.HostAliasesUnderScore + } + dst.Spec.PriorityClassName = params.PriorityClassName + dst.Spec.HostNetwork = params.HostNetwork + dst.Spec.DNSPolicy = params.DNSPolicy + dst.Spec.DNSConfig = params.DNSConfig + dst.Spec.NodeSelector = params.NodeSelector + dst.Spec.SecurityContext = addStrictSecuritySettingsToPod(params) + dst.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds + dst.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints + dst.Spec.ImagePullSecrets = params.ImagePullSecrets + dst.Spec.ReadinessGates = params.ReadinessGates + if params.DisableAutomountServiceAccountToken { + dst.Spec.AutomountServiceAccountToken = ptr.To(false) + } + + cfg := config.MustGetBaseConfig() + if len(cfg.CommonAnnotations) > 0 { + dst.Annotations = labels.Merge(cfg.CommonAnnotations, dst.Annotations) + } + if len(cfg.CommonLabels) > 0 { + dst.Labels = labels.Merge(cfg.CommonLabels, dst.Labels) + } +} diff --git a/internal/controller/operator/factory/build/podtemplate_test.go b/internal/controller/operator/factory/build/podtemplate_test.go new file mode 100644 index 000000000..2f57047e7 --- /dev/null +++ b/internal/controller/operator/factory/build/podtemplate_test.go @@ -0,0 +1,440 @@ +package build + +import ( + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" +) + +func TestPodTemplateParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want corev1.PodSpec) { + t.Helper() + var got corev1.PodTemplateSpec + PodTemplateAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + {MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "zone", Operator: corev1.NodeSelectorOpIn, Values: []string{"us-east"}}}}, + }, + }, + }, + } + tolerations := []corev1.Toleration{{Key: "dedicated", Value: "monitoring", Effect: corev1.TaintEffectNoSchedule}} + + // affinity and tolerations + f( + &vmv1beta1.CommonAppsParams{ + Affinity: affinity, + Tolerations: tolerations, + }, + corev1.PodSpec{ + Affinity: affinity, + Tolerations: tolerations, + }, + ) + + // HostAliases set + f( + &vmv1beta1.CommonAppsParams{ + HostAliases: []corev1.HostAlias{ + {IP: "1.2.3.4", Hostnames: []string{"my.host"}}, + }, + }, + corev1.PodSpec{ + HostAliases: []corev1.HostAlias{ + {IP: "1.2.3.4", Hostnames: []string{"my.host"}}, + }, + }, + ) + + // HostAliasesUnderScore takes precedence over HostAliases + f( + &vmv1beta1.CommonAppsParams{ + HostAliases: []corev1.HostAlias{ + {IP: "1.2.3.4", Hostnames: []string{"old.host"}}, + }, + HostAliasesUnderScore: []corev1.HostAlias{ + {IP: "5.6.7.8", Hostnames: []string{"new.host"}}, + }, + }, + corev1.PodSpec{ + HostAliases: []corev1.HostAlias{ + {IP: "5.6.7.8", Hostnames: []string{"new.host"}}, + }, + }, + ) + + // DisableAutomountServiceAccountToken=true + f( + &vmv1beta1.CommonAppsParams{ + DisableAutomountServiceAccountToken: true, + }, + corev1.PodSpec{ + AutomountServiceAccountToken: ptr.To(false), + }, + ) + + // DisableAutomountServiceAccountToken=false + f( + &vmv1beta1.CommonAppsParams{ + DisableAutomountServiceAccountToken: false, + }, + corev1.PodSpec{}, + ) + + // scheduler, runtime class, priority class, node selector + f( + &vmv1beta1.CommonAppsParams{ + SchedulerName: "custom-scheduler", + RuntimeClassName: ptr.To("gvisor"), + PriorityClassName: "high-priority", + NodeSelector: map[string]string{"disktype": "ssd"}, + }, + corev1.PodSpec{ + SchedulerName: "custom-scheduler", + RuntimeClassName: ptr.To("gvisor"), + PriorityClassName: "high-priority", + NodeSelector: map[string]string{"disktype": "ssd"}, + }, + ) + + // DNS settings + f( + &vmv1beta1.CommonAppsParams{ + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + DNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"8.8.8.8"}, + }, + }, + corev1.PodSpec{ + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + DNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"8.8.8.8"}, + }, + }, + ) + + // image pull secrets and readiness gates + f( + &vmv1beta1.CommonAppsParams{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry-secret"}}, + ReadinessGates: []corev1.PodReadinessGate{{ConditionType: "example.com/ready"}}, + }, + corev1.PodSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry-secret"}}, + ReadinessGates: []corev1.PodReadinessGate{{ConditionType: "example.com/ready"}}, + }, + ) + + // topology spread constraints + f( + &vmv1beta1.CommonAppsParams{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule}, + }, + }, + corev1.PodSpec{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.DoNotSchedule}, + }, + }, + ) + + // termination grace period + f( + &vmv1beta1.CommonAppsParams{ + TerminationGracePeriodSeconds: ptr.To(int64(30)), + }, + corev1.PodSpec{ + TerminationGracePeriodSeconds: ptr.To(int64(30)), + }, + ) + + // host network + f( + &vmv1beta1.CommonAppsParams{ + HostNetwork: true, + }, + corev1.PodSpec{ + HostNetwork: true, + }, + ) +} + +func TestDeploymentAddCommonParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want appsv1.DeploymentSpec) { + t.Helper() + var got appsv1.Deployment + DeploymentAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + // replica count, min ready seconds, revision history + f( + &vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(3)), + MinReadySeconds: 10, + RevisionHistoryLimitCount: ptr.To(int32(5)), + }, + appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(3)), + MinReadySeconds: 10, + RevisionHistoryLimit: ptr.To(int32(5)), + }, + ) + + // pod-level fields are propagated + f( + &vmv1beta1.CommonAppsParams{ + NodeSelector: map[string]string{"env": "prod"}, + ReplicaCount: ptr.To(int32(1)), + }, + appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"env": "prod"}, + }, + }, + }, + ) +} + +func TestStatefulSetAddCommonParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want appsv1.StatefulSetSpec) { + t.Helper() + var got appsv1.StatefulSet + StatefulSetAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + // replica count, min ready seconds, revision history + f( + &vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(2)), + MinReadySeconds: 5, + RevisionHistoryLimitCount: ptr.To(int32(3)), + }, + appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(2)), + MinReadySeconds: 5, + RevisionHistoryLimit: ptr.To(int32(3)), + }, + ) + + // pod-level fields are propagated + f( + &vmv1beta1.CommonAppsParams{ + SchedulerName: "my-scheduler", + ReplicaCount: ptr.To(int32(1)), + }, + appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + SchedulerName: "my-scheduler", + }, + }, + }, + ) +} + +func TestDaemonSetAddCommonParams(t *testing.T) { + f := func(params *vmv1beta1.CommonAppsParams, want appsv1.DaemonSetSpec) { + t.Helper() + var got appsv1.DaemonSet + DaemonSetAddCommonParams(&got, params) + assert.Equal(t, want, got.Spec) + } + + // min ready seconds and revision history + f( + &vmv1beta1.CommonAppsParams{ + MinReadySeconds: 15, + RevisionHistoryLimitCount: ptr.To(int32(2)), + }, + appsv1.DaemonSetSpec{ + MinReadySeconds: 15, + RevisionHistoryLimit: ptr.To(int32(2)), + }, + ) + + // pod-level fields propagated + f( + &vmv1beta1.CommonAppsParams{ + NodeSelector: map[string]string{"role": "worker"}, + }, + appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"role": "worker"}, + }, + }, + }, + ) + + // root-aware security context + f( + &vmv1beta1.CommonAppsParams{ + UseStrictSecurity: ptr.To(true), + }, + appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + SecurityContext: addStrictSecuritySettingsWithRootToPod(&vmv1beta1.CommonAppsParams{ + UseStrictSecurity: ptr.To(true), + }), + }, + }, + }, + ) +} + +func TestPodTemplateAddCommonParams_MergesCommonLabels(t *testing.T) { + cfg := config.MustGetBaseConfig() + defaultCfg := *cfg + defer func() { *config.MustGetBaseConfig() = defaultCfg }() + + f := func(commonLabels map[string]string, initialLabels map[string]string, wantLabels map[string]string) { + t.Helper() + cfg.CommonAnnotations = nil + cfg.CommonLabels = commonLabels + + got := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: initialLabels, + }, + } + + PodTemplateAddCommonParams(&got, &vmv1beta1.CommonAppsParams{}) + + if got.Labels == nil { + got.Labels = map[string]string{} + } + + assert.Equal(t, wantLabels, got.Labels) + } + + // existing + commons + f( + map[string]string{ + "common-label": "common-value", + "existing-label": "should-not-overwrite", + }, + map[string]string{ + "existing-label": "existing-value", + }, + map[string]string{ + "common-label": "common-value", + "existing-label": "existing-value", + }, + ) + + // don't let common rewrite selector-like labels + f( + map[string]string{ + "app.kubernetes.io/name": "common-app", + "app": "common", + }, + map[string]string{ + "app.kubernetes.io/name": "my-app", + "app": "my-app", + }, + map[string]string{ + "app.kubernetes.io/name": "my-app", + "app": "my-app", + }, + ) + + // common selector-like labels added when missing + f( + map[string]string{ + "app.kubernetes.io/name": "common-app", + "app": "common", + }, + map[string]string{}, + map[string]string{ + "app.kubernetes.io/name": "common-app", + "app": "common", + }, + ) + + // mix of selector and regular labels + f( + map[string]string{ + "app.kubernetes.io/name": "common-app", + "common-label": "common-value", + }, + map[string]string{ + "common-label": "existing-value", + }, + map[string]string{ + "app.kubernetes.io/name": "common-app", + "common-label": "existing-value", + }, + ) +} + +func TestPodTemplateAddCommonParams_MergesCommonAnnotations(t *testing.T) { + cfg := config.MustGetBaseConfig() + defaultCfg := *cfg + defer func() { *config.MustGetBaseConfig() = defaultCfg }() + + f := func(commonAnnotations map[string]string, initialAnnotations map[string]string, wantAnnotations map[string]string) { + t.Helper() + cfg.CommonLabels = nil + cfg.CommonAnnotations = commonAnnotations + + got := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: initialAnnotations, + }, + } + + PodTemplateAddCommonParams(&got, &vmv1beta1.CommonAppsParams{}) + + if got.Annotations == nil { + got.Annotations = map[string]string{} + } + + assert.Equal(t, wantAnnotations, got.Annotations) + } + + // existing + commons + f( + map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "should-not-overwrite", + }, + map[string]string{ + "existing-annotation": "existing-value", + }, + map[string]string{ + "common-annotation": "common-value", + "existing-annotation": "existing-value", + }, + ) + + // add annotations when missing + f( + map[string]string{ + "common-annotation": "common-value", + }, + map[string]string{}, + map[string]string{ + "common-annotation": "common-value", + }, + ) +} diff --git a/internal/controller/operator/factory/build/statefulset.go b/internal/controller/operator/factory/build/statefulset.go index a7b19574e..d180fea40 100644 --- a/internal/controller/operator/factory/build/statefulset.go +++ b/internal/controller/operator/factory/build/statefulset.go @@ -2,35 +2,14 @@ package build import ( appsv1 "k8s.io/api/apps/v1" - "k8s.io/utils/ptr" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" ) // StatefulSetAddCommonParams adds common params to given statefulset func StatefulSetAddCommonParams(dst *appsv1.StatefulSet, params *vmv1beta1.CommonAppsParams) { - dst.Spec.Template.Spec.Affinity = params.Affinity - dst.Spec.Template.Spec.Tolerations = params.Tolerations - dst.Spec.Template.Spec.SchedulerName = params.SchedulerName - dst.Spec.Template.Spec.RuntimeClassName = params.RuntimeClassName - dst.Spec.Template.Spec.HostAliases = params.HostAliases - if len(params.HostAliasesUnderScore) > 0 { - dst.Spec.Template.Spec.HostAliases = params.HostAliasesUnderScore - } - dst.Spec.Template.Spec.PriorityClassName = params.PriorityClassName - dst.Spec.Template.Spec.HostNetwork = params.HostNetwork - dst.Spec.Template.Spec.DNSPolicy = params.DNSPolicy - dst.Spec.Template.Spec.DNSConfig = params.DNSConfig - dst.Spec.Template.Spec.NodeSelector = params.NodeSelector - dst.Spec.Template.Spec.SecurityContext = addStrictSecuritySettingsToPod(params) - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds - dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints - dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates + PodTemplateAddCommonParams(&dst.Spec.Template, params) dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.Replicas = params.ReplicaCount dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount - if params.DisableAutomountServiceAccountToken { - dst.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) - } } diff --git a/internal/controller/operator/factory/reconcile/secret.go b/internal/controller/operator/factory/reconcile/secret.go index 3dbbacf20..1e9fcbb09 100644 --- a/internal/controller/operator/factory/reconcile/secret.go +++ b/internal/controller/operator/factory/reconcile/secret.go @@ -18,6 +18,7 @@ import ( func Secret(ctx context.Context, rclient client.Client, newObj *corev1.Secret, prevMeta *metav1.ObjectMeta, owner *metav1.OwnerReference) error { nsn := types.NamespacedName{Name: newObj.Name, Namespace: newObj.Namespace} removeFinalizer := true + rclient.Scheme().Default(newObj) return retryOnConflict(func() error { var existingObj corev1.Secret if err := rclient.Get(ctx, nsn, &existingObj); err != nil { diff --git a/internal/controller/operator/factory/reconcile/service.go b/internal/controller/operator/factory/reconcile/service.go index 0f5c766e4..d3ddbc502 100644 --- a/internal/controller/operator/factory/reconcile/service.go +++ b/internal/controller/operator/factory/reconcile/service.go @@ -23,6 +23,7 @@ import ( // its users responsibility to define it correctly. func Service(ctx context.Context, rclient client.Client, newObj, prevObj *corev1.Service, owner *metav1.OwnerReference) error { svcForReconcile := newObj.DeepCopy() + rclient.Scheme().Default(svcForReconcile) return retryOnConflict(func() error { return reconcileService(ctx, rclient, svcForReconcile, prevObj, owner) }) diff --git a/internal/controller/operator/factory/vlagent/vlagent_test.go b/internal/controller/operator/factory/vlagent/vlagent_test.go index 6589a5f49..b917b210f 100644 --- a/internal/controller/operator/factory/vlagent/vlagent_test.go +++ b/internal/controller/operator/factory/vlagent/vlagent_test.go @@ -18,6 +18,7 @@ import ( vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -25,6 +26,7 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1.VLAgent + cfgMutator func(c *config.BaseOperatorConf) validate func(set *appsv1.StatefulSet) predefinedObjects []runtime.Object } @@ -32,6 +34,14 @@ func TestCreateOrUpdate(t *testing.T) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) ctx := context.TODO() + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) assert.NoError(t, CreateOrUpdate(ctx, o.cr, fclient)) @@ -294,6 +304,71 @@ func TestCreateOrUpdate(t *testing.T) { }, }, }) + + // managed metadata + f(opts{ + cr: &vmv1.VLAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-agent", + Namespace: "default", + }, + Spec: vmv1.VLAgentSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{ + "env": "prod", + }, + Annotations: map[string]string{ + "controller": "true", + }, + }, + RemoteWrite: []vmv1.VLAgentRemoteWriteSpec{ + {URL: "http://remote-write"}, + }, + }, + }, + validate: func(set *appsv1.StatefulSet) { + assert.Equal(t, set.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlagent", + "app.kubernetes.io/instance": "example-agent", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, set.Annotations, map[string]string{ + "controller": "true", + }) + }, + }) + + // common labels + f(opts{ + cr: &vmv1.VLAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-agent", + Namespace: "default", + }, + }, + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{ + "env": "prod", + } + c.CommonAnnotations = map[string]string{ + "controller": "true", + } + }, + validate: func(set *appsv1.StatefulSet) { + assert.Equal(t, set.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlagent", + "app.kubernetes.io/instance": "example-agent", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, set.Annotations, map[string]string{ + "controller": "true", + }) + }, + }) } func TestBuildRemoteWriteArgs(t *testing.T) { diff --git a/internal/controller/operator/factory/vlcluster/vlcluster_test.go b/internal/controller/operator/factory/vlcluster/vlcluster_test.go index 2d4a0a762..449ed560a 100644 --- a/internal/controller/operator/factory/vlcluster/vlcluster_test.go +++ b/internal/controller/operator/factory/vlcluster/vlcluster_test.go @@ -91,35 +91,86 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { // ensure SA created var sa corev1.ServiceAccount - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) assert.Nil(t, sa.Annotations) - assert.Equal(t, sa.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentRoot)) + assert.Equal(t, sa.Labels, map[string]string{ + "app.kubernetes.io/name": "vlcluster", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check insert var dep appsv1.Deployment - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt := dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9481", "-internalselect.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentInsert)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check select - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt = dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9471", "-internalinsert.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentSelect)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt = sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9491", "-storageDataPath=/vlstorage-data"}) assert.Nil(t, sts.Annotations) - assert.Equal(t, sts.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentStorage)) + assert.Equal(t, sts.Labels, map[string]string{ + "app.kubernetes.io/name": "vlstorage", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + + // check services + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlstorage", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -144,7 +195,7 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt := sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-httpListenAddr=:9491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vlstorage-data"}) @@ -180,7 +231,7 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { // check select var d appsv1.Deployment - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &d)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &d)) assert.Len(t, d.Spec.Template.Spec.Containers, 1) cnt := d.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{ @@ -258,9 +309,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentInsert), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -322,9 +379,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -384,9 +447,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlstorage", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -465,9 +534,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vlinsert", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1000", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -553,6 +628,113 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1.VLCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VLClusterSpec{ + VLSelect: &vmv1.VLSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLInsert: &vmv1.VLInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLStorage: &vmv1.VLStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vlselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, set.Annotations, map[string]string{"controller": "true"}) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VLCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VLClusterSpec{ + VLSelect: &vmv1.VLSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLInsert: &vmv1.VLInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VLStorage: &vmv1.VLStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vlselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, set.Annotations, map[string]string{"controller": "true"}) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlselect", + "app.kubernetes.io/part-of": "vlcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func TestCreateOrUpdate_Paused(t *testing.T) { diff --git a/internal/controller/operator/factory/vlsingle/vlsingle.go b/internal/controller/operator/factory/vlsingle/vlsingle.go index fb66a348e..3900a2962 100644 --- a/internal/controller/operator/factory/vlsingle/vlsingle.go +++ b/internal/controller/operator/factory/vlsingle/vlsingle.go @@ -48,8 +48,8 @@ func newPVC(r *vmv1.VLSingle) *corev1.PersistentVolumeClaim { ObjectMeta: metav1.ObjectMeta{ Name: r.PrefixedName(), Namespace: r.Namespace, - Labels: labels.Merge(r.Spec.StorageMetadata.Labels, r.SelectorLabels()), - Annotations: r.Spec.StorageMetadata.Annotations, + Labels: labels.Merge(labels.Merge(r.FinalLabels(), r.Spec.StorageMetadata.Labels), r.SelectorLabels()), + Annotations: labels.Merge(r.FinalAnnotations(), r.Spec.StorageMetadata.Annotations), OwnerReferences: []metav1.OwnerReference{r.AsOwner()}, }, Spec: *r.Spec.Storage, diff --git a/internal/controller/operator/factory/vlsingle/vlsingle_test.go b/internal/controller/operator/factory/vlsingle/vlsingle_test.go index ceb684b71..d66b14168 100644 --- a/internal/controller/operator/factory/vlsingle/vlsingle_test.go +++ b/internal/controller/operator/factory/vlsingle/vlsingle_test.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" @@ -23,8 +24,8 @@ import ( func TestCreateOrUpdateVLSingle(t *testing.T) { type opts struct { cr *vmv1.VLSingle - c *config.BaseOperatorConf - want *appsv1.Deployment + cfgMutator func(c *config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) wantErr bool predefinedObjects []runtime.Object } @@ -32,17 +33,30 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { f := func(o opts) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) - err := CreateOrUpdate(context.TODO(), fclient, o.cr) + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + ctx := context.TODO() + err := CreateOrUpdate(ctx, fclient, o.cr) if o.wantErr { assert.Error(t, err) } else { assert.NoError(t, err) } + if o.validate != nil { + o.validate(ctx, fclient, o.cr) + } } // base gen f(opts{ - c: config.MustGetBaseConfig(), cr: &vmv1.VLSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -70,12 +84,16 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { }, k8stools.NewReadyDeployment("vlsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vlsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vlsingle-base", got.Name) + assert.Equal(t, "default", got.Namespace) + }, }) // base with specific port f(opts{ - c: config.MustGetBaseConfig(), cr: &vmv1.VLSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -105,12 +123,16 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { }, k8stools.NewReadyDeployment("vlsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vlsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vlsingle-base", got.Name) + assert.Equal(t, "default", got.Namespace) + }, }) // with syslog tls config f(opts{ - c: config.MustGetBaseConfig(), cr: &vmv1.VLSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -174,9 +196,66 @@ func TestCreateOrUpdateVLSingle(t *testing.T) { }, k8stools.NewReadyDeployment("vlsingle-base", "default"), }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vlsingle-base", got.Name) + assert.Equal(t, "default", got.Namespace) + }, + }) - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vlsingle-base", Namespace: "default"}}, + // managed metadata + f(opts{ + cr: &vmv1.VLSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VLSingleSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, got.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, got.Annotations, map[string]string{"controller": "true"}) + }, }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VLSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VLSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, got.Labels, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + assert.Equal(t, got.Annotations, map[string]string{"controller": "true"}) + }}) } func TestCreateOrUpdateVLSingle_Paused(t *testing.T) { @@ -206,9 +285,8 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { type opts struct { cr *vmv1.VLSingle c *config.BaseOperatorConf - want *corev1.Service + validate func(*corev1.Service) wantErr bool - wantPortsLen int predefinedObjects []runtime.Object } f := func(o opts) { @@ -228,8 +306,9 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, fclient.Get(ctx, nsn, &got)) - assert.Equal(t, got.Name, o.want.Name) - assert.Len(t, got.Spec.Ports, o.wantPortsLen) + if o.validate != nil { + o.validate(&got) + } } // base service test @@ -241,13 +320,18 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { Namespace: "default", }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vlsingle-logs-1", - Namespace: "default", - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vlsingle-logs-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + assert.Len(t, svc.Spec.Ports, 1) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "logs-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, - wantPortsLen: 1, }) // with extra service nodePort @@ -267,13 +351,18 @@ func TestCreateOrUpdateVLSingleService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vlsingle-logs-1", - Namespace: "default", - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vlsingle-logs-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + assert.Len(t, svc.Spec.Ports, 1) + // verify labels exist and include core operator metadata + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vlsingle", + "app.kubernetes.io/instance": "logs-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, - wantPortsLen: 1, predefinedObjects: []runtime.Object{ &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/controller/operator/factory/vmagent/vmagent_test.go b/internal/controller/operator/factory/vmagent/vmagent_test.go index 4c5b8357c..213bab407 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_test.go +++ b/internal/controller/operator/factory/vmagent/vmagent_test.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -28,6 +29,7 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMAgent + cfgMutator func(*config.BaseOperatorConf) validate func(ctx context.Context, client client.Client, cr *vmv1beta1.VMAgent) wantErr bool predefinedObjects []runtime.Object @@ -37,6 +39,14 @@ func TestCreateOrUpdate(t *testing.T) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) ctx := context.TODO() + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) err := CreateOrUpdate(ctx, o.cr, fclient) @@ -812,6 +822,67 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, ds.Spec.UpdateStrategy.RollingUpdate) }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAgentSpec{ + RemoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{ + {URL: "http://remote-write"}, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, client client.Client, cr *vmv1beta1.VMAgent) { + var set appsv1.Deployment + assert.NoError(t, client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmagent-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAgentSpec{ + RemoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{ + {URL: "http://remote-write"}, + }, + }, + }, + validate: func(ctx context.Context, client client.Client, cr *vmv1beta1.VMAgent) { + var set appsv1.Deployment + assert.NoError(t, client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmagent-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }}) } func TestBuildRemoteWriteArgs(t *testing.T) { @@ -1799,10 +1870,14 @@ func TestBuildRemoteWriteArgs(t *testing.T) { func TestCreateOrUpdateService(t *testing.T) { type opts struct { - cr *vmv1beta1.VMAgent - want func(svc *corev1.Service) error - wantAdditionalService func(svc *corev1.Service) error - predefinedObjects []runtime.Object + cr *vmv1beta1.VMAgent + // legacy functional expectations (kept for compatibility) + want func(svc *corev1.Service) + wantAdditionalService func(svc *corev1.Service) + // new preferred validation hooks (used first if present) + validate func(svc *corev1.Service) + validateAdditional func(svc *corev1.Service) + predefinedObjects []runtime.Object } f := func(o opts) { @@ -1817,11 +1892,20 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, cl.Get(ctx, nsn, &got)) - assert.NoError(t, o.want(&got)) - if o.wantAdditionalService != nil { + // prefer new validate hooks; fall back to legacy want functions when validate is not provided + if o.validate != nil { + o.validate(&got) + } else if o.want != nil { + o.want(&got) + } + if o.validateAdditional != nil { + var additionalSvc corev1.Service + assert.NoError(t, cl.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.Spec.ServiceSpec.NameOrDefault(o.cr.Name)}, &additionalSvc)) + o.validateAdditional(&additionalSvc) + } else if o.wantAdditionalService != nil { var additionalSvc corev1.Service assert.NoError(t, cl.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.Spec.ServiceSpec.NameOrDefault(o.cr.Name)}, &additionalSvc)) - assert.NoError(t, o.wantAdditionalService(&additionalSvc)) + o.wantAdditionalService(&additionalSvc) } } @@ -1833,14 +1917,16 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: "default", }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmagent-base" { - return fmt.Errorf("unexpected name for service: %v", svc.Name) - } - if len(svc.Spec.Ports) != 1 { - return fmt.Errorf("unexpected count for service ports: %v", len(svc.Spec.Ports)) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmagent-base", svc.Name) + assert.Len(t, svc.Spec.Ports, 1) + // ensure operator-managed labels present + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -1880,14 +1966,16 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmagent-base" { - return fmt.Errorf("unexpected name for service: %v", svc.Name) - } - if len(svc.Spec.Ports) != 3 { - return fmt.Errorf("unexpected count for ports, want 3, got: %v", len(svc.Spec.Ports)) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmagent-base", svc.Name) + assert.Len(t, svc.Spec.Ports, 3) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -1934,26 +2022,22 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmagent-base" { - return fmt.Errorf("unexpected name for service: %v", svc.Name) - } - if len(svc.Spec.Ports) != 3 { - return fmt.Errorf("unexpected count for ports, want 3, got: %v", len(svc.Spec.Ports)) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmagent-base", svc.Name) + assert.Len(t, svc.Spec.Ports, 3) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmagent", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, - wantAdditionalService: func(svc *corev1.Service) error { - if len(svc.Spec.Ports) != 1 { - return fmt.Errorf("unexpected count for ports, want 1, got: %v", len(svc.Spec.Ports)) - } - if svc.Spec.Ports[0].NodePort != 8085 { - return fmt.Errorf("unexpected port %v, want 8085", svc.Spec.Ports[0]) - } - if svc.Spec.Ports[0].Protocol != corev1.ProtocolUDP { - return fmt.Errorf("unexpected protocol want udp, got: %v", svc.Spec.Ports[0].Protocol) - } - return nil + validateAdditional: func(svc *corev1.Service) { + // additional service should preserve the explicit port definition + assert.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, int32(8085), svc.Spec.Ports[0].NodePort) + assert.Equal(t, corev1.ProtocolUDP, svc.Spec.Ports[0].Protocol) }, }) } diff --git a/internal/controller/operator/factory/vmalert/vmalert_test.go b/internal/controller/operator/factory/vmalert/vmalert_test.go index 862a11d69..f1bd64e46 100644 --- a/internal/controller/operator/factory/vmalert/vmalert_test.go +++ b/internal/controller/operator/factory/vmalert/vmalert_test.go @@ -2,7 +2,6 @@ package vmalert import ( "context" - "fmt" "strings" "testing" @@ -13,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/config" @@ -23,24 +23,30 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMAlert + cfgMutator func(*config.BaseOperatorConf) cmNames []string predefinedObjects []runtime.Object - validate func(*appsv1.Deployment, *corev1.Secret) + validate func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) } f := func(o opts) { t.Helper() ctx := context.TODO() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) err := CreateOrUpdate(ctx, o.cr, fclient, o.cmNames) assert.NoError(t, err) if o.validate != nil { - var generatedDeploment appsv1.Deployment - assert.NoError(t, fclient.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.PrefixedName()}, &generatedDeploment)) - var generatedTLSSecret corev1.Secret - tlsSecretName := build.ResourceName(build.TLSAssetsResourceKind, o.cr) - assert.NoError(t, fclient.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: tlsSecretName}, &generatedTLSSecret)) - o.validate(&generatedDeploment, &generatedTLSSecret) + o.validate(ctx, fclient, o.cr) } } @@ -95,7 +101,9 @@ func TestCreateOrUpdate(t *testing.T) { predefinedObjects: []runtime.Object{ k8stools.NewReadyDeployment("vmalert-basic-vmalert", "default"), }, - validate: func(d *appsv1.Deployment, s *corev1.Secret) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var d appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &d)) var foundOk bool for _, cnt := range d.Spec.Template.Spec.Containers { if cnt.Name == "vmalert" { @@ -199,10 +207,27 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vmalert-basic-vmalert", "default"), }, - validate: func(d *appsv1.Deployment, s *corev1.Secret) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var s corev1.Secret + tlsSecretName := build.ResourceName(build.TLSAssetsResourceKind, cr) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: tlsSecretName}, &s)) assert.NotEmpty(t, s.Data["default_configmap_datasource-tls_ca"]) assert.NotEmpty(t, s.Data["default_configmap_datasource-tls_cert"]) assert.NotEmpty(t, s.Data["default_datasource-tls_key"]) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "basic-vmalert", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, s.Labels) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "basic-vmalert", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -370,6 +395,91 @@ func TestCreateOrUpdate(t *testing.T) { k8stools.NewReadyDeployment("vmalert-basic-vmalert", "default"), }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAlert{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAlertSpec{ + Notifier: &vmv1beta1.VMAlertNotifierSpec{ + URL: "http://notifier", + }, + Datasource: vmv1beta1.VMAlertDatasourceSpec{ + URL: "http://datasource", + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAlert{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAlertSpec{ + Notifier: &vmv1beta1.VMAlertNotifierSpec{ + URL: "http://notifier", + }, + Datasource: vmv1beta1.VMAlertDatasourceSpec{ + URL: "http://datasource", + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlert) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func TestBuildNotifiers(t *testing.T) { @@ -525,7 +635,7 @@ func TestCreateOrUpdateService(t *testing.T) { type opts struct { cr *vmv1beta1.VMAlert c *config.BaseOperatorConf - want func(svc *corev1.Service) error + validate func(svc *corev1.Service) predefinedObjects []runtime.Object } f := func(o opts) { @@ -540,7 +650,9 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, cl.Get(ctx, nsn, &got)) - assert.NoError(t, o.want(&got)) + if o.validate != nil { + o.validate(&got) + } } // base test @@ -552,11 +664,14 @@ func TestCreateOrUpdateService(t *testing.T) { Name: "base", }, }, - want: func(svc *corev1.Service) error { - if svc.Name != "vmalert-base" { - return fmt.Errorf("unexpected name for vmalert service: %v", svc.Name) - } - return nil + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmalert-base", svc.Name) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmalert", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) } diff --git a/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go b/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go index ecc59bc48..f3ae41135 100644 --- a/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go +++ b/internal/controller/operator/factory/vmalertmanager/alertmanager_test.go @@ -14,8 +14,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -23,7 +25,8 @@ import ( func TestCreateOrUpdateAlertManager(t *testing.T) { type opts struct { cr *vmv1beta1.VMAlertmanager - validate func(set *appsv1.StatefulSet) + cfgMutator func(*config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) wantErr bool predefinedObjects []runtime.Object } @@ -33,6 +36,14 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) ctx := context.TODO() + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } err := CreateOrUpdateAlertManager(ctx, o.cr, fclient) if o.wantErr { assert.Error(t, err) @@ -40,9 +51,7 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { assert.NoError(t, err) } if o.validate != nil { - var got appsv1.StatefulSet - assert.NoError(t, fclient.Get(ctx, types.NamespacedName{Namespace: o.cr.Namespace, Name: o.cr.PrefixedName()}, &got)) - o.validate(&got) + o.validate(ctx, fclient, o.cr) } } @@ -63,7 +72,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Equal(t, set.Name, "vmalertmanager-test-am") assert.Equal(t, set.Spec.Template.Spec.Containers[0].Resources, corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -82,6 +93,15 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { "managed-by": "vm-operator", "main": "system", }) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "main": "system", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "test-am", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -103,7 +123,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Len(t, set.Spec.Template.Spec.Containers, 2) vmaContainer := set.Spec.Template.Spec.Containers[0] assert.Equal(t, vmaContainer.Name, "alertmanager") @@ -141,7 +163,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Equal(t, set.Name, "vmalertmanager-test-am") assert.Len(t, set.Spec.Template.Spec.Volumes, 4) templatesVolume := set.Spec.Template.Spec.Volumes[2] @@ -179,7 +203,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Len(t, set.Spec.Template.Spec.Containers, 2) vmaContainer := set.Spec.Template.Spec.Containers[0] @@ -214,7 +240,9 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, }, }, - validate: func(set *appsv1.StatefulSet) { + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) assert.Len(t, set.Spec.Template.Spec.Containers, 2) vmaContainer := set.Spec.Template.Spec.Containers[0] @@ -232,6 +260,77 @@ func TestCreateOrUpdateAlertManager(t *testing.T) { }, "unexpected cluster peer arguments found") }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAlertmanager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAlertmanagerSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAlertmanager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAlertmanager) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmalertmanager", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func Test_createDefaultAMConfig(t *testing.T) { @@ -263,7 +362,7 @@ func Test_createDefaultAMConfig(t *testing.T) { } var amcfgs vmv1beta1.VMAlertmanagerConfigList - assert.Nil(t, fclient.List(ctx, &amcfgs)) + assert.NoError(t, fclient.List(ctx, &amcfgs)) for _, amcfg := range amcfgs.Items { assert.Equal(t, amcfg.Status.UpdateStatus, vmv1beta1.UpdateStatusOperational) } diff --git a/internal/controller/operator/factory/vmalertmanager/config_test.go b/internal/controller/operator/factory/vmalertmanager/config_test.go index 9d672208d..c25b388fa 100644 --- a/internal/controller/operator/factory/vmalertmanager/config_test.go +++ b/internal/controller/operator/factory/vmalertmanager/config_test.go @@ -2147,7 +2147,7 @@ func Test_UpdateDefaultAMConfig(t *testing.T) { }, }, }) - assert.Nil(t, os.Setenv("WATCH_NAMESPACE", "default")) + assert.NoError(t, os.Setenv("WATCH_NAMESPACE", "default")) } func TestBuildWebConfig(t *testing.T) { diff --git a/internal/controller/operator/factory/vmanomaly/statefulset_test.go b/internal/controller/operator/factory/vmanomaly/statefulset_test.go index cd455e77d..dcfa233ce 100644 --- a/internal/controller/operator/factory/vmanomaly/statefulset_test.go +++ b/internal/controller/operator/factory/vmanomaly/statefulset_test.go @@ -18,6 +18,7 @@ import ( vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -25,6 +26,7 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1.VMAnomaly + cfgMutator func(*config.BaseOperatorConf) validate func(sts *appsv1.StatefulSet, idx int) wantErr bool predefinedObjects []runtime.Object @@ -35,6 +37,14 @@ func TestCreateOrUpdate(t *testing.T) { fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) build.AddDefaults(fclient.Scheme()) fclient.Scheme().Default(o.cr) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } err := CreateOrUpdate(ctx, o.cr, fclient) if o.wantErr { assert.Error(t, err) @@ -299,6 +309,126 @@ schedulers: assert.Equal(t, set.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels["shard-num"], strconv.Itoa(idx)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1.VMAnomaly{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-anomaly", + Namespace: "monitoring", + Annotations: map[string]string{"not": "touch"}, + Labels: map[string]string{"main": "system"}, + }, + Spec: vmv1.VMAnomalySpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + License: &vmv1beta1.License{ + Key: ptr.To("test"), + }, + ConfigRawYaml: ` +reader: + queries: + query_alias2: + expr: vm_metric +writer: + datasource_url: "http://test.com" +models: + model_univariate_1: + class: 'zscore' + z_threshold: 2.5 + queries: ['query_alias2'] +schedulers: + scheduler_periodic_1m: + class: "scheduler.periodic.PeriodicScheduler" + infer_every: 1m + fit_every: 2m + fit_window: 3h +`, + Reader: &vmv1.VMAnomalyReadersSpec{ + DatasourceURL: "http://test.com", + SamplingPeriod: "1m", + }, + Writer: &vmv1.VMAnomalyWritersSpec{ + DatasourceURL: "http://write.endpoint", + }, + }, + }, + validate: func(set *appsv1.StatefulSet, _ int) { + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmanomaly", + "app.kubernetes.io/instance": "test-anomaly", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }, + }) + + // common labels + f(opts{ + cr: &vmv1.VMAnomaly{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-anomaly", + Namespace: "monitoring", + Annotations: map[string]string{"not": "touch"}, + Labels: map[string]string{"main": "system"}, + }, + Spec: vmv1.VMAnomalySpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + License: &vmv1beta1.License{ + Key: ptr.To("test"), + }, + ConfigRawYaml: ` +reader: + queries: + query_alias2: + expr: vm_metric +writer: + datasource_url: "http://test.com" +models: + model_univariate_1: + class: 'zscore' + z_threshold: 2.5 + queries: ['query_alias2'] +schedulers: + scheduler_periodic_1m: + class: "scheduler.periodic.PeriodicScheduler" + infer_every: 1m + fit_every: 2m + fit_window: 3h +`, + Reader: &vmv1.VMAnomalyReadersSpec{ + DatasourceURL: "http://test.com", + SamplingPeriod: "1m", + }, + Writer: &vmv1.VMAnomalyWritersSpec{ + DatasourceURL: "http://write.endpoint", + }, + }, + }, + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + validate: func(set *appsv1.StatefulSet, _ int) { + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmanomaly", + "app.kubernetes.io/instance": "test-anomaly", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + }, + }) } func Test_createDefaultConfig(t *testing.T) { diff --git a/internal/controller/operator/factory/vmauth/vmauth.go b/internal/controller/operator/factory/vmauth/vmauth.go index e6d5144e7..1175a66d3 100644 --- a/internal/controller/operator/factory/vmauth/vmauth.go +++ b/internal/controller/operator/factory/vmauth/vmauth.go @@ -528,13 +528,13 @@ func buildIngressConfig(cr *vmv1beta1.VMAuth) *networkingv1.Ingress { // add user defined routes. spec.Rules = append(spec.Rules, cr.Spec.Ingress.ExtraRules...) spec.TLS = append(spec.TLS, cr.Spec.Ingress.ExtraTLS...) - lbls := labels.Merge(cr.Spec.Ingress.Labels, cr.SelectorLabels()) + lbls := labels.Merge(labels.Merge(cr.FinalLabels(), cr.Spec.Ingress.Labels), cr.SelectorLabels()) return &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: cr.PrefixedName(), Namespace: cr.Namespace, Labels: lbls, - Annotations: cr.Spec.Ingress.Annotations, + Annotations: labels.Merge(cr.FinalAnnotations(), cr.Spec.Ingress.Annotations), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: spec, diff --git a/internal/controller/operator/factory/vmauth/vmauth_test.go b/internal/controller/operator/factory/vmauth/vmauth_test.go index 42bdb791b..573ed4194 100644 --- a/internal/controller/operator/factory/vmauth/vmauth_test.go +++ b/internal/controller/operator/factory/vmauth/vmauth_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" + appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -90,6 +91,23 @@ func TestCreateOrUpdate(t *testing.T) { }, }, }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAuth) { + wantLabels := map[string]string{ + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + } + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, wantLabels, svc.Labels) + var secret corev1.Secret + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.ConfigSecretName()}, &secret)) + assert.Equal(t, wantLabels, secret.Labels) + var httpRoute gwapiv1.HTTPRoute + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &httpRoute)) + assert.Equal(t, wantLabels, httpRoute.Labels) + }, }) // simple-remove-httproute @@ -255,9 +273,14 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -330,9 +353,14 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1000", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -411,6 +439,95 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMAuth{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMAuthSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAuth) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmauth-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + var secret corev1.Secret + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.ConfigSecretName()}, &secret)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, secret.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMAuth{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAuth) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmauth-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + var secret corev1.Secret + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.ConfigSecretName()}, &secret)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmauth", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, secret.Labels) + }}) } func TestMakeSpecForAuthOk(t *testing.T) { diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_test.go index 0d2e5755b..d4821f3f8 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_test.go @@ -435,9 +435,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, nsn, &vpaGot)) vpaExpected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: selectName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: selectName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/part-of": "vmcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -701,6 +707,113 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMClusterSpec{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMClusterSpec{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, svc.Labels) + }}) } func TestCreatOrUpdateClusterServices(t *testing.T) { @@ -776,6 +889,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) // with vmbackup and additional service ports @@ -820,6 +935,7 @@ objectmeta: spec: ports: - name: web-rpc + protocol: TCP port: 8011 targetport: intval: 8011 @@ -850,6 +966,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) @@ -892,6 +1010,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true --- objectmeta: @@ -932,6 +1052,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) // with native and extra service @@ -976,6 +1098,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true --- objectmeta: @@ -1011,6 +1135,10 @@ spec: app.kubernetes.io/name: vmselect managed-by: vm-operator type: LoadBalancer + sessionaffinity: None + externaltrafficpolicy: Cluster + allocateloadbalancernodeports: true + internaltrafficpolicy: Cluster --- objectmeta: name: vmstorage-test @@ -1050,6 +1178,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster publishnotreadyaddresses: true `) f(&vmv1beta1.VMCluster{ @@ -1094,6 +1224,8 @@ spec: app.kubernetes.io/name: vminsert managed-by: vm-operator type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster `) // transit to headless f(&vmv1beta1.VMCluster{ @@ -1152,6 +1284,8 @@ spec: managed-by: vm-operator clusterip: "None" type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster `, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "vminsert-test", @@ -1234,6 +1368,10 @@ spec: app.kubernetes.io/name: vminsert managed-by: vm-operator type: LoadBalancer + sessionaffinity: None + externaltrafficpolicy: Cluster + allocateloadbalancernodeports: true + internaltrafficpolicy: Cluster loadbalancerclass: service.k8s.aws/nlb `, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -1309,6 +1447,8 @@ spec: app.kubernetes.io/name: vmclusterlb-vmauth-balancer managed-by: vm-operator type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster --- objectmeta: name: vminsert-test @@ -1383,6 +1523,10 @@ spec: app.kubernetes.io/name: vminsert managed-by: vm-operator type: LoadBalancer + sessionaffinity: None + externaltrafficpolicy: Cluster + allocateloadbalancernodeports: true + internaltrafficpolicy: Cluster loadbalancerclass: service.k8s.aws/nlb --- objectmeta: @@ -1424,6 +1568,8 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster `, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "vminsert-test", @@ -1499,6 +1645,8 @@ spec: app.kubernetes.io/name: vmclusterlb-vmauth-balancer managed-by: vm-operator type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster --- objectmeta: name: vminsert-test @@ -1540,6 +1688,10 @@ spec: app.kubernetes.io/name: vmclusterlb-vmauth-balancer managed-by: vm-operator type: LoadBalancer + sessionaffinity: None + externaltrafficpolicy: Cluster + allocateloadbalancernodeports: true + internaltrafficpolicy: Cluster loadbalancerclass: service.k8s.aws/nlb --- objectmeta: @@ -1583,6 +1735,9 @@ spec: managed-by: vm-operator clusterip: None type: ClusterIP + sessionaffinity: None + internaltrafficpolicy: Cluster + clusterip: "None" loadbalancerclass: service.k8s.aws/nlb `, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/controller/operator/factory/vmsingle/vmsingle.go b/internal/controller/operator/factory/vmsingle/vmsingle.go index d7163b330..8ad7e6d4c 100644 --- a/internal/controller/operator/factory/vmsingle/vmsingle.go +++ b/internal/controller/operator/factory/vmsingle/vmsingle.go @@ -58,8 +58,8 @@ func makePvc(cr *vmv1beta1.VMSingle) *corev1.PersistentVolumeClaim { ObjectMeta: metav1.ObjectMeta{ Name: cr.PrefixedName(), Namespace: cr.Namespace, - Labels: labels.Merge(cr.Spec.StorageMetadata.Labels, cr.SelectorLabels()), - Annotations: cr.Spec.StorageMetadata.Annotations, + Labels: labels.Merge(labels.Merge(cr.FinalLabels(), cr.Spec.StorageMetadata.Labels), cr.SelectorLabels()), + Annotations: labels.Merge(cr.FinalAnnotations(), cr.Spec.StorageMetadata.Annotations), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: *cr.Spec.Storage, diff --git a/internal/controller/operator/factory/vmsingle/vmsingle_test.go b/internal/controller/operator/factory/vmsingle/vmsingle_test.go index 4e681431d..2e59b7db9 100644 --- a/internal/controller/operator/factory/vmsingle/vmsingle_test.go +++ b/internal/controller/operator/factory/vmsingle/vmsingle_test.go @@ -12,8 +12,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/config" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) @@ -21,14 +23,29 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMSingle - want *appsv1.Deployment + cfgMutator func(*config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) predefinedObjects []runtime.Object } f := func(o opts) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) - assert.NoError(t, CreateOrUpdate(context.TODO(), o.cr, fclient)) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) + ctx := context.TODO() + assert.NoError(t, CreateOrUpdate(ctx, o.cr, fclient)) + if o.validate != nil { + o.validate(ctx, fclient, o.cr) + } } // base-vmsingle-gen @@ -50,7 +67,11 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vmsingle-vmsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vmsingle-vmsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vmsingle-vmsingle-base", got.Name) + }, }) // base-vmsingle-with-ports @@ -78,20 +99,107 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vmsingle-vmsingle-base", "default"), }, - want: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "vmsingle-vmsingle-base", Namespace: "default"}}, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, "vmsingle-vmsingle-base", got.Name) + }, }) + + // managed metadata + f(opts{ + cr: &vmv1beta1.VMSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMSingleSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }}) + + // common labels cannot overwrite standard labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "hacked", + "app.kubernetes.io/instance": "hacked", + "app.kubernetes.io/component": "hacked", + "managed-by": "hacked", + } + }, + cr: &vmv1beta1.VMSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + }}) } func TestCreateOrUpdateService(t *testing.T) { type opts struct { cr *vmv1beta1.VMSingle - want *corev1.Service + validate func(*corev1.Service) predefinedObjects []runtime.Object } f := func(o opts) { t.Helper() fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) + build.AddDefaults(fclient.Scheme()) ctx := context.TODO() assert.NoError(t, createOrUpdateService(ctx, fclient, o.cr, nil)) svc := build.Service(o.cr, o.cr.Spec.Port, nil) @@ -101,8 +209,11 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: svc.Namespace, } assert.NoError(t, fclient.Get(ctx, nsn, &got)) - assert.Equal(t, got.Name, o.want.Name) - assert.Equal(t, got.Spec.Ports, o.want.Spec.Ports) + if o.validate != nil { + o.validate(&got) + } else { + assert.Equal(t, got.Name, svc.Name) + } } // base service test @@ -113,25 +224,30 @@ func TestCreateOrUpdateService(t *testing.T) { Namespace: "default", }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vmsingle-single-1", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, { - Name: "http-alias", - Port: 8428, - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmsingle-single-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + + expectedPorts := []corev1.ServicePort{ + { + Name: "http", + Protocol: "TCP", + TargetPort: intstr.FromInt32(0), }, - }, + { + Name: "http-alias", + Port: 8428, + Protocol: "TCP", + TargetPort: intstr.FromInt32(8428), + }, + } + assert.Equal(t, expectedPorts, svc.Spec.Ports) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "single-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -151,60 +267,27 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vmsingle-single-1", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, { - Name: "http-alias", - Protocol: "TCP", - Port: 8428, - TargetPort: intstr.FromString(""), - }, { - Name: "graphite-tcp", - Protocol: "TCP", - Port: 8053, - TargetPort: intstr.FromInt(8053), - }, { - Name: "graphite-udp", - Protocol: "UDP", - Port: 8053, - TargetPort: intstr.FromInt(8053), - }, { - Name: "influx-tcp", - Protocol: "TCP", - Port: 8051, - TargetPort: intstr.FromInt(8051), - }, { - Name: "influx-udp", - Protocol: "UDP", - Port: 8051, - TargetPort: intstr.FromInt(8051), - }, { - Name: "opentsdb-tcp", - Protocol: "TCP", - Port: 8054, - TargetPort: intstr.FromInt(8054), - }, { - Name: "opentsdb-udp", - Protocol: "UDP", - Port: 8054, - TargetPort: intstr.FromInt(8054), - }, { - Name: "opentsdb-http", - Protocol: "TCP", - Port: 8052, - TargetPort: intstr.FromInt(8052), - }, - }, - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmsingle-single-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + // sanity-check ports count and a couple of representative ports + assert.Len(t, svc.Spec.Ports, 9) + // check graphite tcp present + foundGraphite := false + for _, p := range svc.Spec.Ports { + if p.Name == "graphite-tcp" && p.Port == 8053 { + foundGraphite = true + break + } + } + assert.True(t, foundGraphite) + + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "single-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -224,25 +307,16 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vmsingle-single-1", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, { - Name: "http-alias", - Port: 8428, - Protocol: "TCP", - TargetPort: intstr.FromString(""), - }, - }, - }, + validate: func(svc *corev1.Service) { + assert.Equal(t, "vmsingle-single-1", svc.Name) + assert.Equal(t, "default", svc.Namespace) + assert.Len(t, svc.Spec.Ports, 2) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vmsingle", + "app.kubernetes.io/instance": "single-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, predefinedObjects: []runtime.Object{ &corev1.Service{ diff --git a/internal/controller/operator/factory/vtcluster/cluster_test.go b/internal/controller/operator/factory/vtcluster/cluster_test.go index ecd9030de..f87c25a34 100644 --- a/internal/controller/operator/factory/vtcluster/cluster_test.go +++ b/internal/controller/operator/factory/vtcluster/cluster_test.go @@ -91,35 +91,86 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { // ensure SA created var sa corev1.ServiceAccount - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.GetServiceAccountName(), Namespace: cr.Namespace}, &sa)) assert.Nil(t, sa.Annotations) - assert.Equal(t, sa.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentRoot)) + assert.Equal(t, sa.Labels, map[string]string{ + "app.kubernetes.io/name": "vtcluster", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check insert var dep appsv1.Deployment - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt := dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10481", "-internalselect.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentInsert)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check select - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt = dep.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10471", "-internalinsert.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) assert.Nil(t, dep.Annotations) - assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentSelect)) + assert.Equal(t, dep.Labels, map[string]string{ + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt = sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10491", "-storageDataPath=/vtstorage-data"}) assert.Nil(t, sts.Annotations) - assert.Equal(t, sts.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentStorage)) + assert.Equal(t, sts.Labels, map[string]string{ + "app.kubernetes.io/name": "vtstorage", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }) + + // check services + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &svc)) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtstorage", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) }, }) @@ -144,7 +195,7 @@ func TestCreateOrUpdate(t *testing.T) { validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { // check storage var sts appsv1.StatefulSet - assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt := sts.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-httpListenAddr=:10491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vtstorage-data"}) @@ -213,9 +264,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentInsert), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -277,9 +334,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -339,9 +402,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtstorage", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -420,9 +489,15 @@ func TestCreateOrUpdate(t *testing.T) { assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: vpaName}, &got)) expected := vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ - Name: vpaName, - Namespace: cr.Namespace, - Labels: cr.FinalLabels(component), + Name: vpaName, + Namespace: cr.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "vtinsert", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "test", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, ResourceVersion: "1000", OwnerReferences: []metav1.OwnerReference{{Name: "test", Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}}, }, @@ -508,4 +583,111 @@ func TestCreateOrUpdate(t *testing.T) { assert.True(t, k8serrors.IsNotFound(err)) }, }) + // managed metadata + f(opts{ + cr: &vmv1.VTCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VTClusterSpec{ + Select: &vmv1.VTSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Insert: &vmv1.VTInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Storage: &vmv1.VTStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vtselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vtcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VTCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VTClusterSpec{ + Select: &vmv1.VTSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Insert: &vmv1.VTInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + Storage: &vmv1.VTStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTCluster) { + var set appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vtselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vtcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtselect", + "app.kubernetes.io/part-of": "vtcluster", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) } diff --git a/internal/controller/operator/factory/vtsingle/vtsingle.go b/internal/controller/operator/factory/vtsingle/vtsingle.go index b3d690c4e..9860f4e72 100644 --- a/internal/controller/operator/factory/vtsingle/vtsingle.go +++ b/internal/controller/operator/factory/vtsingle/vtsingle.go @@ -48,8 +48,8 @@ func newPVC(r *vmv1.VTSingle) *corev1.PersistentVolumeClaim { ObjectMeta: metav1.ObjectMeta{ Name: r.PrefixedName(), Namespace: r.Namespace, - Labels: labels.Merge(r.Spec.StorageMetadata.Labels, r.SelectorLabels()), - Annotations: r.Spec.StorageMetadata.Annotations, + Labels: labels.Merge(labels.Merge(r.FinalLabels(), r.Spec.StorageMetadata.Labels), r.SelectorLabels()), + Annotations: labels.Merge(r.FinalAnnotations(), r.Spec.StorageMetadata.Annotations), OwnerReferences: []metav1.OwnerReference{r.AsOwner()}, }, Spec: *r.Spec.Storage, diff --git a/internal/controller/operator/factory/vtsingle/vtsingle_test.go b/internal/controller/operator/factory/vtsingle/vtsingle_test.go index ad44ff663..97322275b 100644 --- a/internal/controller/operator/factory/vtsingle/vtsingle_test.go +++ b/internal/controller/operator/factory/vtsingle/vtsingle_test.go @@ -5,11 +5,13 @@ import ( "testing" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" @@ -21,25 +23,39 @@ import ( func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1.VTSingle - c *config.BaseOperatorConf + cfgMutator func(*config.BaseOperatorConf) + validate func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) wantErr bool predefinedObjects []runtime.Object } - f := func(opts opts) { + f := func(o opts) { t.Helper() - fclient := k8stools.GetTestClientWithObjects(opts.predefinedObjects) - err := CreateOrUpdate(context.TODO(), fclient, opts.cr) - if opts.wantErr { + fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(o.cr) + cfg := config.MustGetBaseConfig() + if o.cfgMutator != nil { + defaultCfg := *cfg + o.cfgMutator(cfg) + defer func() { + *config.MustGetBaseConfig() = defaultCfg + }() + } + ctx := context.TODO() + err := CreateOrUpdate(ctx, fclient, o.cr) + if o.wantErr { assert.Error(t, err) } else { assert.NoError(t, err) } + if o.validate != nil { + o.validate(ctx, fclient, o.cr) + } } // base gen - o := opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -67,12 +83,10 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vtsingle-base", "default"), }, - } - f(o) + }) // base with specific port - o = opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -101,12 +115,10 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vtsingle-base", "default"), }, - } - f(o) + }) // with syslog tls config - o = opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "base", @@ -135,17 +147,68 @@ func TestCreateOrUpdate(t *testing.T) { }, k8stools.NewReadyDeployment("vtsingle-base", "default"), }, - } - f(o) + }) + + // managed metadata + f(opts{ + cr: &vmv1.VTSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1.VTSingleSpec{ + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1.VTSingle{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName()}, &got)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, got.Annotations) + }, + }) } func TestCreateOrUpdateService(t *testing.T) { type opts struct { cr *vmv1.VTSingle - c *config.BaseOperatorConf - want *corev1.Service + validate func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) wantErr bool - wantPortsLen int predefinedObjects []runtime.Object } @@ -158,39 +221,36 @@ func TestCreateOrUpdateService(t *testing.T) { return } assert.NoError(t, err) - svc := build.Service(o.cr, o.cr.Spec.Port, nil) - var got corev1.Service - nsn := types.NamespacedName{ - Name: svc.Name, - Namespace: svc.Namespace, + if o.validate != nil { + o.validate(ctx, fclient, o.cr) } - assert.NoError(t, fclient.Get(ctx, nsn, &got)) - assert.Equal(t, got.Name, o.want.Name) - assert.Len(t, got.Spec.Ports, o.wantPortsLen) } // base service test - o := opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "traces-1", Namespace: "default", }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vtsingle-traces-1", - Namespace: "default", - }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(), Namespace: cr.Namespace}, &got)) + assert.Equal(t, "vtsingle-traces-1", got.Name) + assert.Equal(t, "default", got.Namespace) + assert.Len(t, got.Spec.Ports, 1) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "traces-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) }, - wantPortsLen: 1, - } - f(o) + }) // with extra service nodePort - o = opts{ - c: config.MustGetBaseConfig(), + f(opts{ cr: &vmv1.VTSingle{ ObjectMeta: metav1.ObjectMeta{ Name: "traces-1", @@ -205,13 +265,19 @@ func TestCreateOrUpdateService(t *testing.T) { }, }, }, - want: &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vtsingle-traces-1", - Namespace: "default", - }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1.VTSingle) { + var got corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(), Namespace: cr.Namespace}, &got)) + assert.Equal(t, "vtsingle-traces-1", got.Name) + assert.Equal(t, "default", got.Namespace) + assert.Len(t, got.Spec.Ports, 1) + assert.Equal(t, map[string]string{ + "app.kubernetes.io/name": "vtsingle", + "app.kubernetes.io/instance": "traces-1", + "app.kubernetes.io/component": "monitoring", + "managed-by": "vm-operator", + }, got.Labels) }, - wantPortsLen: 1, predefinedObjects: []runtime.Object{ &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -227,6 +293,5 @@ func TestCreateOrUpdateService(t *testing.T) { Spec: corev1.ServiceSpec{}, }, }, - } - f(o) + }) }