commit cffe13168f3526d91bbb1f51875d1ccffcf7a84f Author: Pengzhan Hao Date: Wed Jan 21 07:00:48 2026 +0000 feat: implement pod-specific secret injection for DaemonSets with automated lifecycle management diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..23b019a --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +bin/ +*.pem +.idea/ +.vscode/ +vendor/ +go.sum diff --git a/Dockerfile.test-client b/Dockerfile.test-client new file mode 100644 index 0000000..fa65650 --- /dev/null +++ b/Dockerfile.test-client @@ -0,0 +1,11 @@ +FROM golang:alpine AS builder +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -o test-client ./cmd/test-client + +FROM alpine:3.18 +WORKDIR /app +COPY --from=builder /app/test-client . +ENTRYPOINT ["./test-client"] diff --git a/Dockerfile.webhook b/Dockerfile.webhook new file mode 100644 index 0000000..6ed6608 --- /dev/null +++ b/Dockerfile.webhook @@ -0,0 +1,11 @@ +FROM golang:alpine AS builder +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -o webhook ./cmd/webhook + +FROM alpine:3.18 +WORKDIR /app +COPY --from=builder /app/webhook . +ENTRYPOINT ["./webhook"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ec36dd6 --- /dev/null +++ b/Makefile @@ -0,0 +1,51 @@ +# Image Registry Configuration +REGISTRY ?= git.pengzhan.dev/haopengzhan/k8s-ds-secret-injection +WEBHOOK_IMAGE = $(REGISTRY)/webhook:latest +CLIENT_IMAGE = $(REGISTRY)/test-client:latest +NAMESPACE = gps-system + +.PHONY: all build build-images push deploy clean certs + +all: build + +build: + go build -o bin/webhook ./cmd/webhook + go build -o bin/test-client ./cmd/test-client + go build -o bin/secret-manager ./cmd/secret-manager + +build-images: + docker build -f Dockerfile.webhook -t $(WEBHOOK_IMAGE) . + docker build -f Dockerfile.test-client -t $(CLIENT_IMAGE) . + +push: build-images + docker push $(WEBHOOK_IMAGE) + docker push $(CLIENT_IMAGE) + +certs: + @echo "Generating self-signed certificates..." + kubectl apply -f deploy/namespace.yaml + openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -days 365 -nodes \ + -subj "/CN=inject-ds-webhook.$(NAMESPACE).svc" \ + -addext "subjectAltName = DNS:inject-ds-webhook.$(NAMESPACE).svc" + kubectl create secret tls inject-ds-webhook-certs --cert=cert.pem --key=key.pem -n $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f - + @echo "Updating CA Bundle in webhook configuration..." + @CA_BUNDLE=$$(cat cert.pem | base64 | tr -d '\n') && \ + sed -i "s/caBundle: .*/caBundle: $$CA_BUNDLE/" deploy/webhook.yaml + +deploy: + kubectl apply -f deploy/namespace.yaml + kubectl apply -f deploy/rbac.yaml + # Ensure images in manifests match our registry + sed -i "s|image: .*/inject-ds-webhook:latest|image: $(WEBHOOK_IMAGE)|" deploy/webhook.yaml + sed -i "s|image: .*/test-client:latest|image: $(CLIENT_IMAGE)|" deploy/test-ds.yaml + kubectl apply -f deploy/webhook.yaml + @echo "Waiting for webhook to be ready..." + kubectl wait --for=condition=available --timeout=60s deployment/inject-ds-webhook -n $(NAMESPACE) + kubectl apply -f deploy/test-ds.yaml + +clean: + kubectl delete -f deploy/test-ds.yaml --ignore-not-found + kubectl delete -f deploy/webhook.yaml --ignore-not-found + kubectl delete -f deploy/rbac.yaml --ignore-not-found + kubectl delete namespace $(NAMESPACE) --ignore-not-found + rm -rf bin/ key.pem cert.pem diff --git a/README.md b/README.md new file mode 100644 index 0000000..7044058 --- /dev/null +++ b/README.md @@ -0,0 +1,80 @@ +# DaemonSet Secret Injector + +A Kubernetes Mutating Webhook that automatically injects a unique, node-specific Secret into Pods belonging to a targeted DaemonSet. + +## Features + +- **Automatic Injection**: Intercepts Pod creation and injects a Secret volume and mount. +- **Node-Specific Data**: Each Pod gets a unique Secret containing information relevant to the node it resides on. +- **Lifecycle Management**: Automatically creates the Secret upon Pod creation and deletes it when the Pod is removed. +- **Node Authorizer Compatibility**: Secret names are derived from the Pod name, ensuring predictable access control. + +## Architecture + +1. **Mutating Webhook**: Intercepts `CREATE` requests for Pods. It calculates the target Node name (from affinity or spec), generates a unique Pod name, creates a Secret, and patches the Pod spec to include the Secret volume. +2. **Pod Controller**: Watches for Pod deletion events and cleans up the associated Secret from the namespace. +3. **Test Client**: A helper utility that uses Node credentials (`kubeconfig`) to verify it can read the injected secret. +4. **Secret Manager**: A CLI tool for managing Kubernetes Secrets from your local machine with automatic decoding. See [cmd/secret-manager/README.md](cmd/secret-manager/README.md) for details. + +## Getting Started + +### Prerequisites + +- Kubernetes cluster (GKE, EKS, Kind, etc.) +- `kubectl` and `docker` +- `openssl` (for certificate generation) + +### Configuration + +The webhook is configured via command-line arguments in the deployment: +- `--target-namespace`: The namespace to monitor (default: `gps-system`). +- `--target-daemonsets`: Comma-separated list of DaemonSet names to inject. + +### Deployment + +1. **Clone the repository**. +2. **(Optional) Configure your registry**: + The default registry is set to `git.pengzhan.dev/haopengzhan/k8s-ds-secret-injection`. If you want to push to your own registry: + ```bash + export REGISTRY=your-registry.com/your-username + ``` +3. **Build and Push images**: + If you are using the default public registry, you can skip the push step if the images are already there. + To push your own images: + ```bash + docker login git.pengzhan.dev + make push + ``` +4. **Setup Certificates**: + ```bash + # Create namespace first + kubectl apply -f deploy/namespace.yaml + # Generate and upload certs + make certs + ``` +5. **Deploy the system**: + ```bash + make deploy REGISTRY=$REGISTRY + ``` + +## Verification + +Check the logs of the test DaemonSet pods to see the successful retrieval of the secret: + +```bash +kubectl logs -n gps-system -l app=test-daemonset +``` + +You should see: +```text +Successfully retrieved secret gps-system/test-daemonset-xxxxx +Key: secret-data, Value: unique-secret-for-test-daemonset-xxxxx-on-node-yyyyy +``` + +## Cleanup + +To remove all resources created by this project: + +```bash +make clean +``` \ No newline at end of file diff --git a/cmd/secret-manager/README.md b/cmd/secret-manager/README.md new file mode 100644 index 0000000..08d843f --- /dev/null +++ b/cmd/secret-manager/README.md @@ -0,0 +1,55 @@ +# Secret Manager + +A simple CLI tool to manage Kubernetes Secrets from your local machine. It handles Base64 encoding/decoding automatically, allowing you to work with plain text values. + +## Features + +- **List**: View all keys and values in a secret (decoded). +- **Add/Update**: Add a new key-value pair or update an existing one using plain text. +- **Delete**: Remove a specific key from a secret. + +## Prerequisites + +- Local `kubeconfig` file (usually at `~/.kube/config`). +- `cluster-admin` or sufficient RBAC permissions to manage secrets in the target namespace. + +## Build + +From the root of the repository: + +```bash +make build +# The binary will be located at bin/secret-manager +``` + +## Usage + +```bash +./bin/secret-manager --secret [options] +``` + +### Options + +- `--namespace`: Namespace of the secret (default: `default`). +- `--secret`: **(Required)** Name of the Kubernetes secret. +- `--op`: Operation to perform: `list`, `add`, or `delete` (default: `list`). +- `--key`: The key to add or delete. +- `--value`: The plain text value to add (required for `add` operation). +- `--kubeconfig`: Path to a custom kubeconfig file. + +### Examples + +**1. List keys and values:** +```bash +./bin/secret-manager --namespace gps-system --secret test-daemonset-7v1v1 --op list +``` + +**2. Add or update a key:** +```bash +./bin/secret-manager --namespace gps-system --secret test-daemonset-7v1v1 --op add --key my-key --value "my-value" +``` + +**3. Delete a key:** +```bash +./bin/secret-manager --namespace gps-system --secret test-daemonset-7v1v1 --op delete --key my-key +``` diff --git a/cmd/secret-manager/main.go b/cmd/secret-manager/main.go new file mode 100644 index 0000000..e4414ad --- /dev/null +++ b/cmd/secret-manager/main.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "path/filepath" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" +) + +func main() { + var kubeconfig *string + if home := homedir.HomeDir(); home != "" { + kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") + } else { + kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file") + } + + var namespace string + var secretName string + var operation string + var key string + var value string + + flag.StringVar(&namespace, "namespace", "default", "Namespace of the secret") + flag.StringVar(&secretName, "secret", "", "Name of the secret") + flag.StringVar(&operation, "op", "list", "Operation: add, delete, list") + flag.StringVar(&key, "key", "", "Key for add/delete operation") + flag.StringVar(&value, "value", "", "Value for add operation") + flag.Parse() + + if secretName == "" { + fmt.Println("Usage: secret-manager --secret [--namespace ] [--op ] [--key ] [--value ]") + os.Exit(1) + } + + config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + if err != nil { + log.Fatalf("Error building kubeconfig: %v", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("Error creating clientset: %v", err) + } + + ctx := context.TODO() + secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + log.Fatalf("Error getting secret %s/%s: %v", namespace, secretName, err) + } + + switch operation { + case "list": + fmt.Printf("Content of secret %s/%s:\n", namespace, secretName) + if len(secret.Data) == 0 { + fmt.Println(" (empty)") + } + for k, v := range secret.Data { + fmt.Printf(" - %s: %s\n", k, string(v)) + } + + case "add": + if key == "" || value == "" { + log.Fatal("Error: --key and --value are required for 'add' operation") + } + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } + secret.Data[key] = []byte(value) + _, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}) + if err != nil { + log.Fatalf("Error updating secret: %v", err) + } + fmt.Printf("Successfully added/updated key '%s' in secret %s/%s\n", key, namespace, secretName) + + case "delete": + if key == "" { + log.Fatal("Error: --key is required for 'delete' operation") + } + if _, ok := secret.Data[key]; !ok { + fmt.Printf("Warning: Key '%s' not found in secret %s/%s\n", key, namespace, secretName) + return + } + delete(secret.Data, key) + _, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}) + if err != nil { + log.Fatalf("Error updating secret: %v", err) + } + fmt.Printf("Successfully deleted key '%s' from secret %s/%s\n", key, namespace, secretName) + + default: + log.Fatalf("Unknown operation: %s. Use list, add, or delete.", operation) + } +} diff --git a/cmd/test-client/main.go b/cmd/test-client/main.go new file mode 100644 index 0000000..31d393f --- /dev/null +++ b/cmd/test-client/main.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const DefaultKubeletConfigPath = "/var/lib/kubelet/kubeconfig" + +func main() { + var namespace string + var secretName string + var kubeconfig string + flag.StringVar(&namespace, "namespace", "", "Namespace of the secret") + flag.StringVar(&secretName, "secret", "", "Name of the secret") + flag.StringVar(&kubeconfig, "kubeconfig", DefaultKubeletConfigPath, "Path to kubeconfig file") + flag.Parse() + + if namespace == "" || secretName == "" { + fmt.Println("Usage: test-client --namespace --secret [--kubeconfig ]") + os.Exit(1) + } + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + log.Fatalf("Failed to get node's kubelet config: %v", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("Failed to create clientset: %v", err) + } + + secret, err := clientset.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + log.Fatalf("Failed to get secret %s/%s: %v", namespace, secretName, err) + } + + fmt.Printf("Successfully retrieved secret %s/%s\n", namespace, secretName) + for k, v := range secret.Data { + fmt.Printf("Key: %s, Value: %s\n", k, string(v)) + } +} + diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go new file mode 100644 index 0000000..4180b73 --- /dev/null +++ b/cmd/webhook/main.go @@ -0,0 +1,191 @@ +package main + +import ( + "context" + "flag" + "os" + "strings" + + "git.pengzhan.dev/k8s-ds-secret-injection/pkg/webhook" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + webhookadmission "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + var targetNS string + var targetDS string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-election", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&targetNS, "target-namespace", "gps-system", "The namespace to monitor.") + flag.StringVar(&targetDS, "target-daemonsets", "", "Comma-separated list of DaemonSet names to monitor.") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: server.Options{ + BindAddress: metricsAddr, + }, + WebhookServer: webhookadmission.NewServer(webhookadmission.Options{ + Port: 9443, + }), + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "inject-ds-lock", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + dsList := strings.Split(targetDS, ",") + for i := range dsList { + dsList[i] = strings.TrimSpace(dsList[i]) + } + + // Register Webhook + hookServer := mgr.GetWebhookServer() + setupLog.Info("registering mutating webhook") + + decoder := admission.NewDecoder(scheme) + + mutator := &webhook.PodMutator{ + Client: mgr.GetClient(), + TargetNS: targetNS, + TargetDSList: dsList, + } + mutator.InjectDecoder(decoder) + + hookServer.Register("/mutate-pod", &admission.Webhook{ + Handler: mutator, + }) + + // Add a simple controller to cleanup secrets when pods are deleted + if err := (&PodController{ + Client: mgr.GetClient(), + TargetNS: targetNS, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Pod") + os.Exit(1) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +// PodController watches Pods and deletes associated secrets on deletion +type PodController struct { + client.Client + TargetNS string +} + +func (r *PodController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.Log.WithValues("pod", req.NamespacedName) + + if req.Namespace != r.TargetNS { + return ctrl.Result{}, nil + } + + pod := &corev1.Pod{} + err := r.Get(ctx, req.NamespacedName, pod) + if err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + // Pod is already gone. We try to delete the secret just in case. + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: req.Namespace, + }, + } + if err := r.Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + // If pod is marked for deletion, cleanup secret and remove finalizer + if !pod.DeletionTimestamp.IsZero() { + log.Info("Pod deleting, cleaning up associated secret") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name, + Namespace: pod.Namespace, + }, + } + if err := r.Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + + // Remove finalizer + newFinalizers := []string{} + changed := false + for _, f := range pod.Finalizers { + if f == "inject-ds-webhook.example.com/cleanup" { + changed = true + continue + } + newFinalizers = append(newFinalizers, f) + } + if changed { + pod.Finalizers = newFinalizers + if err := r.Update(ctx, pod); err != nil { + return ctrl.Result{}, err + } + } + } + + return ctrl.Result{}, nil +} + +func (r *PodController) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + Complete(r) +} diff --git a/deploy/namespace.yaml b/deploy/namespace.yaml new file mode 100644 index 0000000..fc18b1f --- /dev/null +++ b/deploy/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gps-system + labels: + kubernetes.io/metadata.name: gps-system diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml new file mode 100644 index 0000000..4492eab --- /dev/null +++ b/deploy/rbac.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: inject-ds-webhook + namespace: gps-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: inject-ds-webhook-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete", "get", "list", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: inject-ds-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: inject-ds-webhook-role +subjects: +- kind: ServiceAccount + name: inject-ds-webhook + namespace: gps-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-ds-sa + namespace: gps-system \ No newline at end of file diff --git a/deploy/test-ds.yaml b/deploy/test-ds.yaml new file mode 100644 index 0000000..a38ad1b --- /dev/null +++ b/deploy/test-ds.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: test-daemonset + namespace: gps-system +spec: + selector: + matchLabels: + app: test-daemonset + template: + metadata: + labels: + app: test-daemonset + spec: + serviceAccountName: test-ds-sa + terminationGracePeriodSeconds: 0 + containers: + - name: test-client + image: REPO_PLACEHOLDER/test-client:latest + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: ["/bin/sh", "-c"] + args: + - | + /app/test-client --namespace=$(POD_NAMESPACE) --secret=$(POD_NAME) + echo "Test client finished. Sleeping forever..." + sleep infinity + volumeMounts: + - name: kubeconfig + mountPath: /var/lib/kubelet/kubeconfig + readOnly: true + - name: gke-bin + mountPath: /home/kubernetes/bin + readOnly: true + - name: pki + mountPath: /etc/srv/kubernetes/pki + readOnly: true + - name: kubelet-pki + mountPath: /var/lib/kubelet/pki + readOnly: true + volumes: + - name: kubeconfig + hostPath: + path: /var/lib/kubelet/kubeconfig + type: File + - name: gke-bin + hostPath: + path: /home/kubernetes/bin + type: Directory + - name: pki + hostPath: + path: /etc/srv/kubernetes/pki + type: Directory + - name: kubelet-pki + hostPath: + path: /var/lib/kubelet/pki + type: Directory diff --git a/deploy/webhook.yaml b/deploy/webhook.yaml new file mode 100644 index 0000000..535fadb --- /dev/null +++ b/deploy/webhook.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inject-ds-webhook + namespace: gps-system +spec: + replicas: 1 + selector: + matchLabels: + app: inject-ds-webhook + template: + metadata: + labels: + app: inject-ds-webhook + spec: + serviceAccountName: inject-ds-webhook + containers: + - name: webhook + image: REPO_PLACEHOLDER/inject-ds-webhook:latest + args: + - --target-namespace=gps-system + - --target-daemonsets=test-daemonset + ports: + - containerPort: 9443 + name: webhook-api + volumeMounts: + - name: webhook-certs + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: inject-ds-webhook-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: inject-ds-webhook + namespace: gps-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app: inject-ds-webhook +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: inject-ds-webhook +webhooks: +- name: inject-ds.example.com + clientConfig: + service: + name: inject-ds-webhook + namespace: gps-system + path: "/mutate-pod" + caBundle: Cg== + rules: + - operations: ["CREATE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + admissionReviewVersions: ["v1"] + sideEffects: None + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gps-system + objectSelector: + matchExpressions: + - key: app + operator: NotIn + values: ["inject-ds-webhook"] \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ad241f8 --- /dev/null +++ b/go.mod @@ -0,0 +1,68 @@ +module git.pengzhan.dev/k8s-ds-secret-injection + +go 1.25 + +require ( + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/client-go v0.28.3 + sigs.k8s.io/controller-runtime v0.16.3 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.25.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.28.3 // indirect + k8s.io/component-base v0.28.3 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/pkg/webhook/pod_mutator.go b/pkg/webhook/pod_mutator.go new file mode 100644 index 0000000..7e34eb3 --- /dev/null +++ b/pkg/webhook/pod_mutator.go @@ -0,0 +1,165 @@ +package webhook + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func randomString(n int) string { + var letters = []rune("abcdefghijklmnopqrstuvwxyz0123456789") + s := make([]rune, n) + for i := range s { + s[i] = letters[rand.Intn(len(letters))] + } + return string(s) +} + +var log = ctrl.Log.WithName("pod-mutator") + +// PodMutator mutates Pods +type PodMutator struct { + Client client.Client + decoder *admission.Decoder + TargetNS string + TargetDSList []string +} + +// PodMutator implements admission.DecoderInjector. +// A decoder will be automatically injected. + +// InjectDecoder injects the decoder. +func (a *PodMutator) InjectDecoder(d *admission.Decoder) error { + a.decoder = d + return nil +} + +// Handle handles admission requests. +func (a *PodMutator) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + + err := a.decoder.Decode(req, pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // Check namespace + if pod.Namespace == "" { + pod.Namespace = req.Namespace + } + + if pod.Namespace != a.TargetNS { + return admission.Allowed("not in target namespace") + } + + // Check if it belongs to target DaemonSet + isTarget := false + dsName := "" + for _, targetDS := range a.TargetDSList { + for _, owner := range pod.OwnerReferences { + if owner.Kind == "DaemonSet" && owner.Name == targetDS { + isTarget = true + dsName = targetDS + break + } + } + if isTarget { + break + } + } + + if !isTarget { + return admission.Allowed("not a target daemonset pod") + } + + // Extract NodeName from affinity if not set (common for DaemonSets) + nodeName := pod.Spec.NodeName + if nodeName == "" && pod.Spec.Affinity != nil && + pod.Spec.Affinity.NodeAffinity != nil && + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + for _, term := range pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + for _, expr := range term.MatchFields { + if expr.Key == "metadata.name" && expr.Operator == corev1.NodeSelectorOpIn && len(expr.Values) > 0 { + nodeName = expr.Values[0] + break + } + } + } + } + + // Requirement: Secret name is the same as pod name. + // Since pod.Name is empty during CREATE, we must assign it. + if pod.Name == "" { + pod.Name = pod.GenerateName + randomString(5) + } + secretName := pod.Name + + // Add finalizer to ensure our controller can cleanup the secret before pod is gone + pod.Finalizers = append(pod.Finalizers, "inject-ds-webhook.example.com/cleanup") + + log.Info("Mutating pod", "node", nodeName, "podName", pod.Name, "secretName", secretName, "ds", dsName) + + // Create Secret + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: pod.Namespace, + Labels: map[string]string{ + "injected-by": "inject-ds-webhook", + }, + }, + StringData: map[string]string{ + "node-name": nodeName, + "pod-name": pod.Name, + "secret-data": fmt.Sprintf("unique-secret-for-%s-on-%s", pod.Name, nodeName), + }, + } + + err = a.Client.Create(ctx, secret) + if err != nil && !apierrors.IsAlreadyExists(err) { + return admission.Errored(http.StatusInternalServerError, err) + } + + // Mutate Pod to add volume + volumeName := "node-secret-volume" + + // Add volume + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + }, + }, + }) + + // Add volume mount to all containers + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: "/etc/node-secret", + ReadOnly: true, + }) + } + + marshaledPod, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} \ No newline at end of file