add subpath to configmap mount

Author: cunningr-cisco

cmd/cisco-vk/run.go

@@ -211,7 +211,7 @@ func runVirtualKubelet(cmd *cobra.Command, args []string) error {
 	if keyFile == "" {
 		keyFile = tlsutil.DefaultKeyFile
 	}
-	tlsCfg, err := tlsutil.EnsureTLSConfig(certFile, keyFile, appCfg.Device.Address)
+	tlsCfg, err := tlsutil.EnsureTLSConfig(certFile, keyFile, tlsutil.DefaultGenCertFile, tlsutil.DefaultGenKeyFile, appCfg.Device.Address)
 	if err != nil {
 		return fmt.Errorf("failed to configure kubelet TLS: %w", err)
 	}

internal/controller/ciscodevice_controller.go

@@ -46,6 +46,10 @@ const (
 	configMountPath = "/etc/virtual-kubelet"
 	// configFileName is the key used inside the ConfigMap.
 	configFileName = "config.yaml"
+	// tlsGenMountPath is the writable directory where the VK process writes
+	// its self-signed TLS certificate when no Secret-provided cert is found.
+	// An emptyDir is mounted here so the path is writable even on a RORFS.
+	varLibMountPath = "/var/lib/virtual-kubelet"
 	// DefaultImage is the default container image for the VK deployment.
 	DefaultImage = "ghcr.io/cisco/virtual-kubelet-cisco:latest"
 	// DefaultServiceAccount is the shared service account used by all VK deployments.
@@ -182,9 +186,14 @@ func (r *CiscoDeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 					VolumeMounts: []corev1.VolumeMount{
 						{
 							Name:      "device-config",
-							MountPath: configMountPath,
+							MountPath: configMountPath + "/" + configFileName,
+							SubPath:   configFileName,
 							ReadOnly:  true,
 						},
+						{
+							Name:      "tls-gen",
+							MountPath: varLibMountPath,
+						},
 					},
 				},
 			},
@@ -199,6 +208,15 @@ func (r *CiscoDeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 						},
 					},
 				},
+				{
+					// emptyDir provides a writable scratch space for the
+					// self-signed TLS cert generated at startup. Using an
+					// explicit emptyDir ensures this works on a RORFS.
+					Name: "tls-gen",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
 			},
 			// Use shared service account with VK RBAC permissions
 			ServiceAccountName: serviceAccount,

internal/controller/ciscodevice_controller_test.go

@@ -149,8 +149,8 @@ func TestReconcile_CreatesDeployment(t *testing.T) {
 	if !found {
 		t.Errorf("expected --nodename router-b in container args, got %v", args)
 	}
-	if len(deploy.Spec.Template.Spec.Containers[0].VolumeMounts) != 1 {
-		t.Errorf("expected 1 volume mount, got %d", len(deploy.Spec.Template.Spec.Containers[0].VolumeMounts))
+	if len(deploy.Spec.Template.Spec.Containers[0].VolumeMounts) != 2 {
+		t.Errorf("expected 2 volume mounts (device-config, tls-gen), got %d", len(deploy.Spec.Template.Spec.Containers[0].VolumeMounts))
 	}
 }
 

internal/tlsutil/tls.go

@@ -32,40 +32,49 @@ import (
 )
 
 const (
-	// DefaultCertFile is the default path for the kubelet TLS certificate.
-	// A Kubernetes Secret of type kubernetes.io/tls mounted at
-	// /etc/virtual-kubelet/tls/ will populate this path automatically,
-	// allowing Secret-provided certs to take precedence over generated ones.
+	// DefaultCertFile is the preferred path for the kubelet TLS certificate.
+	// Mount a Kubernetes Secret of type kubernetes.io/tls here to have it
+	// take precedence over any generated certificate.
 	DefaultCertFile = "/etc/virtual-kubelet/tls/tls.crt"
 
-	// DefaultKeyFile is the default path for the kubelet TLS private key.
+	// DefaultKeyFile is the preferred path for the kubelet TLS private key.
 	DefaultKeyFile = "/etc/virtual-kubelet/tls/tls.key"
+
+	// DefaultGenCertFile is the writable fallback path where a self-signed
+	// certificate is written when the preferred path does not exist.
+	// /var/lib is writable in containers even when /etc is read-only.
+	DefaultGenCertFile = "/var/lib/virtual-kubelet/tls/tls.crt"
+
+	// DefaultGenKeyFile is the writable fallback path for the generated key.
+	DefaultGenKeyFile = "/var/lib/virtual-kubelet/tls/tls.key"
 )
 
 // EnsureTLSConfig returns a *tls.Config for the kubelet HTTPS listener.
 //
 // Behaviour:
 //   - If both certFile and keyFile exist on disk, they are loaded and returned.
-//     This lets credentials provisioned via a Kubernetes Secret mount take
-//     precedence automatically -- no restart or reconfiguration required.
-//   - If neither file exists, a self-signed ECDSA certificate is generated,
-//     written to certFile/keyFile (parent directories are created as needed),
-//     and returned. Persisting the files keeps the TLS fingerprint stable
-//     across restarts.
-//   - If exactly one file is present, an error is returned; this typically
-//     indicates a partial or misconfigured Secret mount.
+//     This is the expected path when a Kubernetes Secret of type
+//     kubernetes.io/tls is mounted at the certFile/keyFile location.
+//   - If neither certFile nor keyFile exists, a self-signed ECDSA certificate
+//     is generated and written to genCertFile/genKeyFile (parent directories
+//     are created as needed). Using a separate writable path for generation
+//     allows certFile/keyFile to live under a read-only ConfigMap mount.
+//     Persisting the generated files keeps the fingerprint stable across
+//     restarts.
+//   - If exactly one of certFile/keyFile is present, an error is returned;
+//     this typically indicates a partial or misconfigured Secret mount.
 //
 // deviceAddr is added as a Subject Alternative Name when generating a
 // self-signed certificate so that both local and remote health checks pass.
-func EnsureTLSConfig(certFile, keyFile, deviceAddr string) (*tls.Config, error) {
+func EnsureTLSConfig(certFile, keyFile, genCertFile, genKeyFile, deviceAddr string) (*tls.Config, error) {
 	certExists := fileExists(certFile)
 	keyExists := fileExists(keyFile)
 
 	switch {
 	case certExists && keyExists:
 		return loadTLSConfig(certFile, keyFile)
 	case !certExists && !keyExists:
-		return generateAndWrite(certFile, keyFile, deviceAddr)
+		return generateAndWrite(genCertFile, genKeyFile, deviceAddr)
 	default:
 		return nil, fmt.Errorf(
 			"tls misconfiguration: only one of %q / %q is present; provide both or neither",

internal/tlsutil/tls_test.go

@@ -45,7 +45,7 @@ func TestEnsureTLSConfig_Generate(t *testing.T) {
 	certFile := filepath.Join(dir, "tls.crt")
 	keyFile := filepath.Join(dir, "tls.key")
 
-	cfg, err := EnsureTLSConfig(certFile, keyFile, "")
+	cfg, err := EnsureTLSConfig(certFile, keyFile, certFile, keyFile, "")
 	if err != nil {
 		t.Fatalf("EnsureTLSConfig() error = %v", err)
 	}
@@ -64,6 +64,8 @@ func TestEnsureTLSConfig_Generate(t *testing.T) {
 func TestEnsureTLSConfig_IPAddressSAN(t *testing.T) {
 	dir := t.TempDir()
 	cfg, err := EnsureTLSConfig(
+		filepath.Join(dir, "tls.crt"),
+		filepath.Join(dir, "tls.key"),
 		filepath.Join(dir, "tls.crt"),
 		filepath.Join(dir, "tls.key"),
 		"192.168.1.1",
@@ -91,6 +93,8 @@ func TestEnsureTLSConfig_IPAddressSAN(t *testing.T) {
 func TestEnsureTLSConfig_DNSSANHostname(t *testing.T) {
 	dir := t.TempDir()
 	cfg, err := EnsureTLSConfig(
+		filepath.Join(dir, "tls.crt"),
+		filepath.Join(dir, "tls.key"),
 		filepath.Join(dir, "tls.crt"),
 		filepath.Join(dir, "tls.key"),
 		"device.example.com",
@@ -121,13 +125,13 @@ func TestEnsureTLSConfig_LoadFromDisk(t *testing.T) {
 	keyFile := filepath.Join(dir, "tls.key")
 
 	// First call generates and writes the files.
-	first, err := EnsureTLSConfig(certFile, keyFile, "")
+	first, err := EnsureTLSConfig(certFile, keyFile, certFile, keyFile, "")
 	if err != nil {
 		t.Fatalf("first EnsureTLSConfig() error = %v", err)
 	}
 
 	// Second call must load from disk — same cert bytes.
-	second, err := EnsureTLSConfig(certFile, keyFile, "")
+	second, err := EnsureTLSConfig(certFile, keyFile, certFile, keyFile, "")
 	if err != nil {
 		t.Fatalf("second EnsureTLSConfig() error = %v", err)
 	}
@@ -152,7 +156,7 @@ func TestEnsureTLSConfig_MisconfigOnlyOneFile(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err := EnsureTLSConfig(certFile, keyFile, "")
+	_, err := EnsureTLSConfig(certFile, keyFile, certFile, keyFile, "")
 	if err == nil {
 		t.Fatal("expected an error, got nil")
 	}