feat: implement pod-specific secret injection for DaemonSets with automated lifecycle management
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
bin/
|
||||
*.pem
|
||||
.idea/
|
||||
.vscode/
|
||||
vendor/
|
||||
go.sum
|
||||
@@ -0,0 +1,11 @@
|
||||
FROM golang:alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o test-client ./cmd/test-client
|
||||
|
||||
FROM alpine:3.18
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/test-client .
|
||||
ENTRYPOINT ["./test-client"]
|
||||
@@ -0,0 +1,11 @@
|
||||
FROM golang:alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o webhook ./cmd/webhook
|
||||
|
||||
FROM alpine:3.18
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/webhook .
|
||||
ENTRYPOINT ["./webhook"]
|
||||
@@ -0,0 +1,51 @@
|
||||
# Image Registry Configuration
|
||||
REGISTRY ?= us-docker.pkg.dev/your-project/your-repo
|
||||
WEBHOOK_IMAGE = $(REGISTRY)/inject-ds-webhook:latest
|
||||
CLIENT_IMAGE = $(REGISTRY)/test-client:latest
|
||||
NAMESPACE = gps-system
|
||||
|
||||
.PHONY: all build build-images push deploy clean certs
|
||||
|
||||
all: build
|
||||
|
||||
build:
|
||||
go build -o bin/webhook ./cmd/webhook
|
||||
go build -o bin/test-client ./cmd/test-client
|
||||
go build -o bin/secret-manager ./cmd/secret-manager
|
||||
|
||||
build-images:
|
||||
docker build -f Dockerfile.webhook -t $(WEBHOOK_IMAGE) .
|
||||
docker build -f Dockerfile.test-client -t $(CLIENT_IMAGE) .
|
||||
|
||||
push: build-images
|
||||
docker push $(WEBHOOK_IMAGE)
|
||||
docker push $(CLIENT_IMAGE)
|
||||
|
||||
certs:
|
||||
@echo "Generating self-signed certificates..."
|
||||
kubectl apply -f deploy/namespace.yaml
|
||||
openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -days 365 -nodes \
|
||||
-subj "/CN=inject-ds-webhook.$(NAMESPACE).svc" \
|
||||
-addext "subjectAltName = DNS:inject-ds-webhook.$(NAMESPACE).svc"
|
||||
kubectl create secret tls inject-ds-webhook-certs --cert=cert.pem --key=key.pem -n $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f -
|
||||
@echo "Updating CA Bundle in webhook configuration..."
|
||||
@CA_BUNDLE=$$(cat cert.pem | base64 | tr -d '\n') && \
|
||||
sed -i "s/caBundle: .*/caBundle: $$CA_BUNDLE/" deploy/webhook.yaml
|
||||
|
||||
deploy:
|
||||
kubectl apply -f deploy/namespace.yaml
|
||||
kubectl apply -f deploy/rbac.yaml
|
||||
# Ensure images in manifests match our registry
|
||||
sed -i "s|image: .*/inject-ds-webhook:latest|image: $(WEBHOOK_IMAGE)|" deploy/webhook.yaml
|
||||
sed -i "s|image: .*/test-client:latest|image: $(CLIENT_IMAGE)|" deploy/test-ds.yaml
|
||||
kubectl apply -f deploy/webhook.yaml
|
||||
@echo "Waiting for webhook to be ready..."
|
||||
kubectl wait --for=condition=available --timeout=60s deployment/inject-ds-webhook -n $(NAMESPACE)
|
||||
kubectl apply -f deploy/test-ds.yaml
|
||||
|
||||
clean:
|
||||
kubectl delete -f deploy/test-ds.yaml --ignore-not-found
|
||||
kubectl delete -f deploy/webhook.yaml --ignore-not-found
|
||||
kubectl delete -f deploy/rbac.yaml --ignore-not-found
|
||||
kubectl delete namespace $(NAMESPACE) --ignore-not-found
|
||||
rm -rf bin/ key.pem cert.pem
|
||||
@@ -0,0 +1,76 @@
|
||||
# DaemonSet Secret Injector
|
||||
|
||||
A Kubernetes Mutating Webhook that automatically injects a unique, node-specific Secret into Pods belonging to a targeted DaemonSet.
|
||||
|
||||
## Features
|
||||
|
||||
- **Automatic Injection**: Intercepts Pod creation and injects a Secret volume and mount.
|
||||
- **Node-Specific Data**: Each Pod gets a unique Secret containing information relevant to the node it resides on.
|
||||
- **Lifecycle Management**: Automatically creates the Secret upon Pod creation and deletes it when the Pod is removed.
|
||||
- **Node Authorizer Compatibility**: Secret names are derived from the Pod name, ensuring predictable access control.
|
||||
|
||||
## Architecture
|
||||
|
||||
1. **Mutating Webhook**: Intercepts `CREATE` requests for Pods. It calculates the target Node name (from affinity or spec), generates a unique Pod name, creates a Secret, and patches the Pod spec to include the Secret volume.
|
||||
2. **Pod Controller**: Watches for Pod deletion events and cleans up the associated Secret from the namespace.
|
||||
3. **Test Client**: A helper utility that uses Node credentials (`kubeconfig`) to verify it can read the injected secret.
|
||||
4. **Secret Manager**: A CLI tool for managing Kubernetes Secrets from your local machine with automatic decoding. See [cmd/secret-manager/README.md](cmd/secret-manager/README.md) for details.
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Kubernetes cluster (GKE, EKS, Kind, etc.)
|
||||
- `kubectl` and `docker`
|
||||
- `openssl` (for certificate generation)
|
||||
|
||||
### Configuration
|
||||
|
||||
The webhook is configured via command-line arguments in the deployment:
|
||||
- `--target-namespace`: The namespace to monitor (default: `gps-system`).
|
||||
- `--target-daemonsets`: Comma-separated list of DaemonSet names to inject.
|
||||
|
||||
### Deployment
|
||||
|
||||
1. **Clone the repository**.
|
||||
2. **Configure your registry**:
|
||||
```bash
|
||||
export REGISTRY=us-docker.pkg.dev/your-project/your-repo
|
||||
```
|
||||
3. **Build and Push images**:
|
||||
```bash
|
||||
make push REGISTRY=$REGISTRY
|
||||
```
|
||||
4. **Setup Certificates**:
|
||||
```bash
|
||||
# Create namespace first
|
||||
kubectl apply -f deploy/namespace.yaml
|
||||
# Generate and upload certs
|
||||
make certs
|
||||
```
|
||||
5. **Deploy the system**:
|
||||
```bash
|
||||
make deploy REGISTRY=$REGISTRY
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
Check the logs of the test DaemonSet pods to see the successful retrieval of the secret:
|
||||
|
||||
```bash
|
||||
kubectl logs -n gps-system -l app=test-daemonset
|
||||
```
|
||||
|
||||
You should see:
|
||||
```text
|
||||
Successfully retrieved secret gps-system/test-daemonset-xxxxx
|
||||
Key: secret-data, Value: unique-secret-for-test-daemonset-xxxxx-on-node-yyyyy
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
||||
To remove all resources created by this project:
|
||||
|
||||
```bash
|
||||
make clean
|
||||
```
|
||||
@@ -0,0 +1,55 @@
|
||||
# Secret Manager
|
||||
|
||||
A simple CLI tool to manage Kubernetes Secrets from your local machine. It handles Base64 encoding/decoding automatically, allowing you to work with plain text values.
|
||||
|
||||
## Features
|
||||
|
||||
- **List**: View all keys and values in a secret (decoded).
|
||||
- **Add/Update**: Add a new key-value pair or update an existing one using plain text.
|
||||
- **Delete**: Remove a specific key from a secret.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Local `kubeconfig` file (usually at `~/.kube/config`).
|
||||
- `cluster-admin` or sufficient RBAC permissions to manage secrets in the target namespace.
|
||||
|
||||
## Build
|
||||
|
||||
From the root of the repository:
|
||||
|
||||
```bash
|
||||
make build
|
||||
# The binary will be located at bin/secret-manager
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
./bin/secret-manager --secret <name> [options]
|
||||
```
|
||||
|
||||
### Options
|
||||
|
||||
- `--namespace`: Namespace of the secret (default: `default`).
|
||||
- `--secret`: **(Required)** Name of the Kubernetes secret.
|
||||
- `--op`: Operation to perform: `list`, `add`, or `delete` (default: `list`).
|
||||
- `--key`: The key to add or delete.
|
||||
- `--value`: The plain text value to add (required for `add` operation).
|
||||
- `--kubeconfig`: Path to a custom kubeconfig file.
|
||||
|
||||
### Examples
|
||||
|
||||
**1. List keys and values:**
|
||||
```bash
|
||||
./bin/secret-manager --namespace gps-system --secret test-daemonset-7v1v1 --op list
|
||||
```
|
||||
|
||||
**2. Add or update a key:**
|
||||
```bash
|
||||
./bin/secret-manager --namespace gps-system --secret test-daemonset-7v1v1 --op add --key my-key --value "my-value"
|
||||
```
|
||||
|
||||
**3. Delete a key:**
|
||||
```bash
|
||||
./bin/secret-manager --namespace gps-system --secret test-daemonset-7v1v1 --op delete --key my-key
|
||||
```
|
||||
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/client-go/util/homedir"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var kubeconfig *string
|
||||
if home := homedir.HomeDir(); home != "" {
|
||||
kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
|
||||
} else {
|
||||
kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
|
||||
}
|
||||
|
||||
var namespace string
|
||||
var secretName string
|
||||
var operation string
|
||||
var key string
|
||||
var value string
|
||||
|
||||
flag.StringVar(&namespace, "namespace", "default", "Namespace of the secret")
|
||||
flag.StringVar(&secretName, "secret", "", "Name of the secret")
|
||||
flag.StringVar(&operation, "op", "list", "Operation: add, delete, list")
|
||||
flag.StringVar(&key, "key", "", "Key for add/delete operation")
|
||||
flag.StringVar(&value, "value", "", "Value for add operation")
|
||||
flag.Parse()
|
||||
|
||||
if secretName == "" {
|
||||
fmt.Println("Usage: secret-manager --secret <name> [--namespace <ns>] [--op <add|delete|list>] [--key <k>] [--value <v>]")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Error building kubeconfig: %v", err)
|
||||
}
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating clientset: %v", err)
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting secret %s/%s: %v", namespace, secretName, err)
|
||||
}
|
||||
|
||||
switch operation {
|
||||
case "list":
|
||||
fmt.Printf("Content of secret %s/%s:\n", namespace, secretName)
|
||||
if len(secret.Data) == 0 {
|
||||
fmt.Println(" (empty)")
|
||||
}
|
||||
for k, v := range secret.Data {
|
||||
fmt.Printf(" - %s: %s\n", k, string(v))
|
||||
}
|
||||
|
||||
case "add":
|
||||
if key == "" || value == "" {
|
||||
log.Fatal("Error: --key and --value are required for 'add' operation")
|
||||
}
|
||||
if secret.Data == nil {
|
||||
secret.Data = make(map[string][]byte)
|
||||
}
|
||||
secret.Data[key] = []byte(value)
|
||||
_, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Fatalf("Error updating secret: %v", err)
|
||||
}
|
||||
fmt.Printf("Successfully added/updated key '%s' in secret %s/%s\n", key, namespace, secretName)
|
||||
|
||||
case "delete":
|
||||
if key == "" {
|
||||
log.Fatal("Error: --key is required for 'delete' operation")
|
||||
}
|
||||
if _, ok := secret.Data[key]; !ok {
|
||||
fmt.Printf("Warning: Key '%s' not found in secret %s/%s\n", key, namespace, secretName)
|
||||
return
|
||||
}
|
||||
delete(secret.Data, key)
|
||||
_, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
log.Fatalf("Error updating secret: %v", err)
|
||||
}
|
||||
fmt.Printf("Successfully deleted key '%s' from secret %s/%s\n", key, namespace, secretName)
|
||||
|
||||
default:
|
||||
log.Fatalf("Unknown operation: %s. Use list, add, or delete.", operation)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
)
|
||||
|
||||
const DefaultKubeletConfigPath = "/var/lib/kubelet/kubeconfig"
|
||||
|
||||
func main() {
|
||||
var namespace string
|
||||
var secretName string
|
||||
var kubeconfig string
|
||||
flag.StringVar(&namespace, "namespace", "", "Namespace of the secret")
|
||||
flag.StringVar(&secretName, "secret", "", "Name of the secret")
|
||||
flag.StringVar(&kubeconfig, "kubeconfig", DefaultKubeletConfigPath, "Path to kubeconfig file")
|
||||
flag.Parse()
|
||||
|
||||
if namespace == "" || secretName == "" {
|
||||
fmt.Println("Usage: test-client --namespace <ns> --secret <name> [--kubeconfig <path>]")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get node's kubelet config: %v", err)
|
||||
}
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create clientset: %v", err)
|
||||
}
|
||||
|
||||
secret, err := clientset.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get secret %s/%s: %v", namespace, secretName, err)
|
||||
}
|
||||
|
||||
fmt.Printf("Successfully retrieved secret %s/%s\n", namespace, secretName)
|
||||
for k, v := range secret.Data {
|
||||
fmt.Printf("Key: %s, Value: %s\n", k, string(v))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.pengzhan.dev/k8s-ds-secret-injection/pkg/webhook"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
_ "k8s.io/client-go/plugin/pkg/client/auth"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/healthz"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
webhookadmission "sigs.k8s.io/controller-runtime/pkg/webhook"
|
||||
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
|
||||
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
|
||||
)
|
||||
|
||||
var (
|
||||
scheme = runtime.NewScheme()
|
||||
setupLog = ctrl.Log.WithName("setup")
|
||||
)
|
||||
|
||||
func init() {
|
||||
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
|
||||
// +kubebuilder:scaffold:scheme
|
||||
}
|
||||
|
||||
func main() {
|
||||
var metricsAddr string
|
||||
var enableLeaderElection bool
|
||||
var probeAddr string
|
||||
var targetNS string
|
||||
var targetDS string
|
||||
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
|
||||
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
|
||||
flag.BoolVar(&enableLeaderElection, "leader-election", false,
|
||||
"Enable leader election for controller manager. "+
|
||||
"Enabling this will ensure there is only one active controller manager.")
|
||||
flag.StringVar(&targetNS, "target-namespace", "gps-system", "The namespace to monitor.")
|
||||
flag.StringVar(&targetDS, "target-daemonsets", "", "Comma-separated list of DaemonSet names to monitor.")
|
||||
opts := zap.Options{
|
||||
Development: true,
|
||||
}
|
||||
opts.BindFlags(flag.CommandLine)
|
||||
flag.Parse()
|
||||
|
||||
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
|
||||
|
||||
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
|
||||
Scheme: scheme,
|
||||
Metrics: server.Options{
|
||||
BindAddress: metricsAddr,
|
||||
},
|
||||
WebhookServer: webhookadmission.NewServer(webhookadmission.Options{
|
||||
Port: 9443,
|
||||
}),
|
||||
HealthProbeBindAddress: probeAddr,
|
||||
LeaderElection: enableLeaderElection,
|
||||
LeaderElectionID: "inject-ds-lock",
|
||||
})
|
||||
if err != nil {
|
||||
setupLog.Error(err, "unable to start manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
dsList := strings.Split(targetDS, ",")
|
||||
for i := range dsList {
|
||||
dsList[i] = strings.TrimSpace(dsList[i])
|
||||
}
|
||||
|
||||
// Register Webhook
|
||||
hookServer := mgr.GetWebhookServer()
|
||||
setupLog.Info("registering mutating webhook")
|
||||
|
||||
decoder := admission.NewDecoder(scheme)
|
||||
|
||||
mutator := &webhook.PodMutator{
|
||||
Client: mgr.GetClient(),
|
||||
TargetNS: targetNS,
|
||||
TargetDSList: dsList,
|
||||
}
|
||||
mutator.InjectDecoder(decoder)
|
||||
|
||||
hookServer.Register("/mutate-pod", &admission.Webhook{
|
||||
Handler: mutator,
|
||||
})
|
||||
|
||||
// Add a simple controller to cleanup secrets when pods are deleted
|
||||
if err := (&PodController{
|
||||
Client: mgr.GetClient(),
|
||||
TargetNS: targetNS,
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "Pod")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
|
||||
setupLog.Error(err, "unable to set up health check")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
|
||||
setupLog.Error(err, "unable to set up ready check")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
setupLog.Info("starting manager")
|
||||
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
|
||||
setupLog.Error(err, "problem running manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// PodController watches Pods and deletes associated secrets on deletion
|
||||
type PodController struct {
|
||||
client.Client
|
||||
TargetNS string
|
||||
}
|
||||
|
||||
func (r *PodController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
log := ctrl.Log.WithValues("pod", req.NamespacedName)
|
||||
|
||||
if req.Namespace != r.TargetNS {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
pod := &corev1.Pod{}
|
||||
err := r.Get(ctx, req.NamespacedName, pod)
|
||||
if err != nil {
|
||||
if !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
// Pod is already gone. We try to delete the secret just in case.
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: req.Name,
|
||||
Namespace: req.Namespace,
|
||||
},
|
||||
}
|
||||
if err := r.Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// If pod is marked for deletion, cleanup secret and remove finalizer
|
||||
if !pod.DeletionTimestamp.IsZero() {
|
||||
log.Info("Pod deleting, cleaning up associated secret")
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: pod.Name,
|
||||
Namespace: pod.Namespace,
|
||||
},
|
||||
}
|
||||
if err := r.Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
// Remove finalizer
|
||||
newFinalizers := []string{}
|
||||
changed := false
|
||||
for _, f := range pod.Finalizers {
|
||||
if f == "inject-ds-webhook.example.com/cleanup" {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
newFinalizers = append(newFinalizers, f)
|
||||
}
|
||||
if changed {
|
||||
pod.Finalizers = newFinalizers
|
||||
if err := r.Update(ctx, pod); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *PodController) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&corev1.Pod{}).
|
||||
Complete(r)
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: gps-system
|
||||
labels:
|
||||
kubernetes.io/metadata.name: gps-system
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: inject-ds-webhook
|
||||
namespace: gps-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: inject-ds-webhook-role
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "watch", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "delete", "get", "list", "patch", "update"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: inject-ds-webhook-binding
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: inject-ds-webhook-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: inject-ds-webhook
|
||||
namespace: gps-system
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: test-ds-sa
|
||||
namespace: gps-system
|
||||
@@ -0,0 +1,68 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: test-daemonset
|
||||
namespace: gps-system
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: test-daemonset
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: test-daemonset
|
||||
spec:
|
||||
serviceAccountName: test-ds-sa
|
||||
terminationGracePeriodSeconds: 0
|
||||
containers:
|
||||
- name: test-client
|
||||
image: us-docker.pkg.dev/haopengzhan-gke-dev/haopengzhan-gke-dev/test-client:latest
|
||||
env:
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
/app/test-client --namespace=$(POD_NAMESPACE) --secret=$(POD_NAME)
|
||||
echo "Test client finished. Sleeping forever..."
|
||||
sleep infinity
|
||||
volumeMounts:
|
||||
- name: kubeconfig
|
||||
mountPath: /var/lib/kubelet/kubeconfig
|
||||
readOnly: true
|
||||
- name: gke-bin
|
||||
mountPath: /home/kubernetes/bin
|
||||
readOnly: true
|
||||
- name: pki
|
||||
mountPath: /etc/srv/kubernetes/pki
|
||||
readOnly: true
|
||||
- name: kubelet-pki
|
||||
mountPath: /var/lib/kubelet/pki
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: kubeconfig
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/kubeconfig
|
||||
type: File
|
||||
- name: gke-bin
|
||||
hostPath:
|
||||
path: /home/kubernetes/bin
|
||||
type: Directory
|
||||
- name: pki
|
||||
hostPath:
|
||||
path: /etc/srv/kubernetes/pki
|
||||
type: Directory
|
||||
- name: kubelet-pki
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pki
|
||||
type: Directory
|
||||
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: inject-ds-webhook
|
||||
namespace: gps-system
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: inject-ds-webhook
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: inject-ds-webhook
|
||||
spec:
|
||||
serviceAccountName: inject-ds-webhook
|
||||
containers:
|
||||
- name: webhook
|
||||
image: us-docker.pkg.dev/haopengzhan-gke-dev/haopengzhan-gke-dev/inject-ds-webhook:latest
|
||||
args:
|
||||
- --target-namespace=gps-system
|
||||
- --target-daemonsets=test-daemonset
|
||||
ports:
|
||||
- containerPort: 9443
|
||||
name: webhook-api
|
||||
volumeMounts:
|
||||
- name: webhook-certs
|
||||
mountPath: /tmp/k8s-webhook-server/serving-certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: webhook-certs
|
||||
secret:
|
||||
secretName: inject-ds-webhook-certs
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: inject-ds-webhook
|
||||
namespace: gps-system
|
||||
spec:
|
||||
ports:
|
||||
- port: 443
|
||||
targetPort: 9443
|
||||
selector:
|
||||
app: inject-ds-webhook
|
||||
---
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: MutatingWebhookConfiguration
|
||||
metadata:
|
||||
name: inject-ds-webhook
|
||||
webhooks:
|
||||
- name: inject-ds.example.com
|
||||
clientConfig:
|
||||
service:
|
||||
name: inject-ds-webhook
|
||||
namespace: gps-system
|
||||
path: "/mutate-pod"
|
||||
caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURaVENDQWsyZ0F3SUJBZ0lVUDhweFlpRlpIMVdzajBUNDk2dnNKdWF6TGJFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0t6RXBNQ2NHQTFVRUF3d2dhVzVxWldOMExXUnpMWGRsWW1odmIyc3VaM0J6TFhONWMzUmxiUzV6ZG1NdwpIaGNOTWpZd01USXhNRFl6TkRJM1doY05NamN3TVRJeE1EWXpOREkzV2pBck1Ta3dKd1lEVlFRRERDQnBibXBsClkzUXRaSE10ZDJWaWFHOXZheTVuY0hNdGMzbHpkR1Z0TG5OMll6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQUQKZ2dFUEFEQ0NBUW9DZ2dFQkFLUWQ3T3p5QU12ZkJWalVTb1h4eW5qTmc4UzM3WkhKaFNCQm9wZjhOa0N4OFlESQpQOXhYN3ZyMjFMOWEvWHlvRG9ocHFTUWQwUkptZ0czelBXMnFuQ3RZeUJvQm1iSk8weXBCWVpaRERXaU9ad0loCjcrVmpvbFJrdHIzdkRjbnZZekJUenBEa2JDUkphbHZlNEZuZlZLeXp4RmF6a2svMi9RMEVLeU5ncEFhY0E5SmYKZUp5RWcrUHpCUUR6ZzVTeEdZTS9qbGg2cU5XWHYrTmJHcVM4aVNHd0NwOUdJVWtoakRHRmUzcnBDc3BwaFQ2cAp5U3d6T1pPcmtwUy9zY1Z5ZWQ0VlFXdnFWd2VpV1U2am9QZjk1WTVFZG5wbEYvRGFpV1IzblpQTnZDUkY3ODlMCnVsRTZicE84UVRmc3oxcVZ4RVpVekE2eVFuQjQvaUh6Sm5ZMTRwTUNBd0VBQWFPQmdEQitNQjBHQTFVZERnUVcKQkJTZkFnQWNzbk4wSFU1YkJ6MjM4c25DazBRVDVUQWZCZ05WSFNNRUdEQVdnQlNmQWdBY3NuTjBIVTViQnoyMwo4c25DazBRVDVUQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01Dc0dBMVVkRVFRa01DS0NJR2x1YW1WamRDMWtjeTEzClpXSm9iMjlyTG1kd2N5MXplWE4wWlcwdWMzWmpNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUFjZGtGQ2Z2V2cKRVZLS2xEZmY5ZnlucGVKZ0w0TzNVZWFhcis3OUd4VldPME1jUWJMeW9WNHd4TXJ4eDg5ZG9rbHpHKzV2MTlEZgpoTnhtYjhZRGVZMERNZVZyaitGZllQaW9zUFhKWThjNlN5YldCNVRvcHZnTXVYVHcyTUJKSlNLalI1OWFLcGZsCklWL2xNa3V5NmZLc2J1VWVMdEtRSzZ5aG1qekQxa2FaYjJBZ2dTUDhJOHZDODNmbDlTWCtEbHJaWENDUHJITmMKa2d4Nk1GSFFOR2xmV1AvSnpOTlY1aEMwOHZ1VkVaSlUxM3lKTVYxbk1mWFQxVk1ZTWxUQytndm9mU1F5ZjIxMwprL0RJU0E3ZVBHaXpTY0dFNGNodkRzcTlyaGVUQkx3Nnd0cW1iRk1QRHBzaDBVelZGVThyT1A0QkJmZ0dQc24xCldHY1Ruam5OdUtrNgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
|
||||
rules:
|
||||
- operations: ["CREATE"]
|
||||
apiGroups: [""]
|
||||
apiVersions: ["v1"]
|
||||
resources: ["pods"]
|
||||
admissionReviewVersions: ["v1"]
|
||||
sideEffects: None
|
||||
namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: gps-system
|
||||
objectSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: NotIn
|
||||
values: ["inject-ds-webhook"]
|
||||
@@ -0,0 +1,68 @@
|
||||
module git.pengzhan.dev/k8s-ds-secret-injection
|
||||
|
||||
go 1.25
|
||||
|
||||
require (
|
||||
k8s.io/api v0.28.3
|
||||
k8s.io/apimachinery v0.28.3
|
||||
k8s.io/client-go v0.28.3
|
||||
sigs.k8s.io/controller-runtime v0.16.3
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
|
||||
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-logr/logr v1.2.4 // indirect
|
||||
github.com/go-logr/zapr v1.2.4 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.6 // indirect
|
||||
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
||||
github.com/go-openapi/swag v0.22.3 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/gnostic-models v0.6.8 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/gofuzz v1.2.0 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/imdario/mergo v0.3.6 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/prometheus/client_golang v1.16.0 // indirect
|
||||
github.com/prometheus/client_model v0.4.0 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.10.1 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.25.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
|
||||
golang.org/x/net v0.17.0 // indirect
|
||||
golang.org/x/oauth2 v0.8.0 // indirect
|
||||
golang.org/x/sys v0.13.0 // indirect
|
||||
golang.org/x/term v0.13.0 // indirect
|
||||
golang.org/x/text v0.13.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/protobuf v1.30.0 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/apiextensions-apiserver v0.28.3 // indirect
|
||||
k8s.io/component-base v0.28.3 // indirect
|
||||
k8s.io/klog/v2 v2.100.1 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
|
||||
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
|
||||
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
|
||||
sigs.k8s.io/yaml v1.3.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,165 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func randomString(n int) string {
|
||||
var letters = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
|
||||
s := make([]rune, n)
|
||||
for i := range s {
|
||||
s[i] = letters[rand.Intn(len(letters))]
|
||||
}
|
||||
return string(s)
|
||||
}
|
||||
|
||||
var log = ctrl.Log.WithName("pod-mutator")
|
||||
|
||||
// PodMutator mutates Pods
|
||||
type PodMutator struct {
|
||||
Client client.Client
|
||||
decoder *admission.Decoder
|
||||
TargetNS string
|
||||
TargetDSList []string
|
||||
}
|
||||
|
||||
// PodMutator implements admission.DecoderInjector.
|
||||
// A decoder will be automatically injected.
|
||||
|
||||
// InjectDecoder injects the decoder.
|
||||
func (a *PodMutator) InjectDecoder(d *admission.Decoder) error {
|
||||
a.decoder = d
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handle handles admission requests.
|
||||
func (a *PodMutator) Handle(ctx context.Context, req admission.Request) admission.Response {
|
||||
pod := &corev1.Pod{}
|
||||
|
||||
err := a.decoder.Decode(req, pod)
|
||||
if err != nil {
|
||||
return admission.Errored(http.StatusBadRequest, err)
|
||||
}
|
||||
|
||||
// Check namespace
|
||||
if pod.Namespace == "" {
|
||||
pod.Namespace = req.Namespace
|
||||
}
|
||||
|
||||
if pod.Namespace != a.TargetNS {
|
||||
return admission.Allowed("not in target namespace")
|
||||
}
|
||||
|
||||
// Check if it belongs to target DaemonSet
|
||||
isTarget := false
|
||||
dsName := ""
|
||||
for _, targetDS := range a.TargetDSList {
|
||||
for _, owner := range pod.OwnerReferences {
|
||||
if owner.Kind == "DaemonSet" && owner.Name == targetDS {
|
||||
isTarget = true
|
||||
dsName = targetDS
|
||||
break
|
||||
}
|
||||
}
|
||||
if isTarget {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !isTarget {
|
||||
return admission.Allowed("not a target daemonset pod")
|
||||
}
|
||||
|
||||
// Extract NodeName from affinity if not set (common for DaemonSets)
|
||||
nodeName := pod.Spec.NodeName
|
||||
if nodeName == "" && pod.Spec.Affinity != nil &&
|
||||
pod.Spec.Affinity.NodeAffinity != nil &&
|
||||
pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
|
||||
for _, term := range pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
|
||||
for _, expr := range term.MatchFields {
|
||||
if expr.Key == "metadata.name" && expr.Operator == corev1.NodeSelectorOpIn && len(expr.Values) > 0 {
|
||||
nodeName = expr.Values[0]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Requirement: Secret name is the same as pod name.
|
||||
// Since pod.Name is empty during CREATE, we must assign it.
|
||||
if pod.Name == "" {
|
||||
pod.Name = pod.GenerateName + randomString(5)
|
||||
}
|
||||
secretName := pod.Name
|
||||
|
||||
// Add finalizer to ensure our controller can cleanup the secret before pod is gone
|
||||
pod.Finalizers = append(pod.Finalizers, "inject-ds-webhook.example.com/cleanup")
|
||||
|
||||
log.Info("Mutating pod", "node", nodeName, "podName", pod.Name, "secretName", secretName, "ds", dsName)
|
||||
|
||||
// Create Secret
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: secretName,
|
||||
Namespace: pod.Namespace,
|
||||
Labels: map[string]string{
|
||||
"injected-by": "inject-ds-webhook",
|
||||
},
|
||||
},
|
||||
StringData: map[string]string{
|
||||
"node-name": nodeName,
|
||||
"pod-name": pod.Name,
|
||||
"secret-data": fmt.Sprintf("unique-secret-for-%s-on-%s", pod.Name, nodeName),
|
||||
},
|
||||
}
|
||||
|
||||
err = a.Client.Create(ctx, secret)
|
||||
if err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
return admission.Errored(http.StatusInternalServerError, err)
|
||||
}
|
||||
|
||||
// Mutate Pod to add volume
|
||||
volumeName := "node-secret-volume"
|
||||
|
||||
// Add volume
|
||||
pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
|
||||
Name: volumeName,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: secretName,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// Add volume mount to all containers
|
||||
for i := range pod.Spec.Containers {
|
||||
pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, corev1.VolumeMount{
|
||||
Name: volumeName,
|
||||
MountPath: "/etc/node-secret",
|
||||
ReadOnly: true,
|
||||
})
|
||||
}
|
||||
|
||||
marshaledPod, err := json.Marshal(pod)
|
||||
if err != nil {
|
||||
return admission.Errored(http.StatusInternalServerError, err)
|
||||
}
|
||||
|
||||
return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
|
||||
}
|
||||
Reference in New Issue
Block a user