Kubernetes Interview Questions & Answers part 04

# Deployment — stateless web app
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: web-app
  template:
    metadata:
      labels:
        app: web-app
    spec:
      containers:
      - name: web
        image: nginx:1.25
        ports:
        - containerPort: 80
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 256Mi

# StatefulSet — PostgreSQL with stable identity
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: postgres
spec:
  serviceName: postgres-headless   # Required: headless service name
  replicas: 3
  selector:
    matchLabels:
      app: postgres
  template:
    metadata:
      labels:
        app: postgres
    spec:
      containers:
      - name: postgres
        image: postgres:15
        env:
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: pg-secret
              key: password
        volumeMounts:
        - name: data
          mountPath: /var/lib/postgresql/data
  # Each pod gets its own dedicated PVC
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: fast-ssd
      resources:
        requests:
          storage: 20Gi

# Namespace per team+environment
apiVersion: v1
kind: Namespace
metadata:
  name: team-alpha-prod
  labels:
    team: alpha
    environment: production
    cost-center: "cc-1234"

# ResourceQuota — cap total resources per namespace
apiVersion: v1
kind: ResourceQuota
metadata:
  name: team-alpha-quota
  namespace: team-alpha-prod
spec:
  hard:
    requests.cpu: "10"
    requests.memory: 20Gi
    limits.cpu: "20"
    limits.memory: 40Gi
    pods: "50"
    services: "10"
    persistentvolumeclaims: "20"
    count/deployments.apps: "20"

# LimitRange — set default container resource requests/limits
apiVersion: v1
kind: LimitRange
metadata:
  name: default-limits
  namespace: team-alpha-prod
spec:
  limits:
  - type: Container
    defaultRequest:
      cpu: 100m
      memory: 128Mi
    default:
      cpu: 500m
      memory: 256Mi
    max:
      cpu: "4"
      memory: 4Gi
    min:
      cpu: 50m
      memory: 64Mi

# Set default namespace for current session
kubectl config set-context --current --namespace=team-alpha-prod

# Cross-namespace DNS format
# <service>.<namespace>.svc.cluster.local
curl http://db-service.team-alpha-prod.svc.cluster.local:5432

# Check etcd health
kubectl exec -n kube-system etcd-master -- etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  endpoint health

# Check etcd member list
etcdctl member list --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# Take a snapshot backup
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-$(date +%Y%m%d-%H%M%S).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# Verify snapshot
etcdctl snapshot status /backup/etcd-20240101.db --write-out=table

# Restore from snapshot
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd-20240101.db \
  --data-dir=/var/lib/etcd-restored \
  --name=master \
  --initial-cluster=master=https://127.0.0.1:2380 \
  --initial-advertise-peer-urls=https://127.0.0.1:2380

# PodDisruptionBudget — prevents too many pods being evicted at once
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: web-app-pdb
  namespace: production
spec:
  minAvailable: 2          # Always keep at least 2 pods running
  # OR: maxUnavailable: 1  # Allow at most 1 pod down at a time
  selector:
    matchLabels:
      app: web-app

# Node maintenance workflow
kubectl cordon node-1         # Mark unschedulable — no new pods land here
kubectl drain node-1 \
  --ignore-daemonsets \       # Ignore DaemonSet pods (can't be moved)
  --delete-emptydir-data \    # Delete pods using emptyDir volumes
  --grace-period=60           # Give pods 60s to terminate gracefully
# Do maintenance...
kubectl uncordon node-1       # Mark schedulable again

# Check node conditions
kubectl describe node node-1 | grep -A 20 "Conditions:"

# apply — idempotent, tracks last-applied-configuration annotation
kubectl apply -f deployment.yaml

# create — fails if already exists
kubectl create -f deployment.yaml

# replace — deletes and recreates (loses rollout history)
kubectl replace -f deployment.yaml
kubectl replace --force -f deployment.yaml   # Delete first, then create

# patch — merge patch
kubectl patch deployment web-app \
  -p '{"spec":{"replicas":5}}'

# patch — JSON patch (precise)
kubectl patch deployment web-app \
  --type=json \
  -p='[{"op":"replace","path":"/spec/replicas","value":5}]'

# patch — strategic merge patch (aware of K8s object structure)
kubectl patch deployment web-app \
  --type=strategic \
  -p '{"spec":{"template":{"spec":{"containers":[{"name":"web","image":"nginx:1.26"}]}}}}'

# Diff before applying
kubectl diff -f deployment.yaml

# Deployment — pods get these labels
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  selector:
    matchLabels:
      app: web-app        # ReplicaSet selects pods with this label
      tier: frontend
  template:
    metadata:
      labels:
        app: web-app      # Pod label — MUST match selector above
        tier: frontend
        version: v2.0
        env: production

# Service — routes to pods matching selector
apiVersion: v1
kind: Service
metadata:
  name: web-app-svc
spec:
  selector:
    app: web-app          # Matches pods with this label
    tier: frontend        # AND this label
  ports:
  - port: 80
    targetPort: 8080

# Diagnose empty endpoints
kubectl get endpoints web-app-svc
# NAME          ENDPOINTS   AGE
# web-app-svc   <none>      5m  ← PROBLEM

# Check service selector
kubectl get svc web-app-svc -o jsonpath='{.spec.selector}'
# {"app":"web-app","tier":"frontend"}

# Check what labels pods actually have
kubectl get pods --show-labels | grep web-app
# web-app-xxxx  Running  app=web-app,tier=backend  ← MISMATCH!

# Fix: update pod labels to match service selector
kubectl label pod web-app-xxxx tier=frontend --overwrite

# Other label operations
kubectl get pods -l 'app=web-app,env=production'   # Select by multiple labels
kubectl get pods -l 'version in (v1.0, v2.0)'      # Set-based selector
kubectl get pods -l 'env notin (dev, staging)'      # Exclusion selector
kubectl label pod web-app-xxxx version-             # Remove a label

# Step 1: Get overview
kubectl get pods -n production
# NAME           READY   STATUS             RESTARTS   AGE
# app-xxx-yyy    0/1     CrashLoopBackOff   8          12m

# Step 2: Describe pod — read Events section carefully
kubectl describe pod app-xxx-yyy -n production
# Look for:
#   Exit Code (137=OOM, 1=app error, 139=segfault, 143=SIGTERM)
#   Last State: reason, exitCode, finishedAt
#   Events: Failed to pull image, Failed to mount volume, etc.

# Step 3: Check CURRENT logs
kubectl logs app-xxx-yyy -n production

# Step 4: Check PREVIOUS container logs (before crash — most useful!)
kubectl logs app-xxx-yyy -n production --previous

# Step 5: Check resource pressure
kubectl top pod app-xxx-yyy -n production
kubectl describe pod app-xxx-yyy | grep -A5 "Limits\|Requests"

# Step 6: Override entrypoint to keep container alive for debugging
kubectl run debug \
  --image=your-app-image:tag \
  --restart=Never \
  --command -- sleep 3600
kubectl exec -it debug -- /bin/sh

# Step 7: Check events across namespace
kubectl get events -n production --sort-by='.lastTimestamp' | tail -20

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
  namespace: production
spec:
  replicas: 4
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1           # Create 1 extra pod above desired count
      maxUnavailable: 0     # Never go below desired count (zero-downtime)
  selector:
    matchLabels:
      app: web-app
  template:
    metadata:
      labels:
        app: web-app
    spec:
      containers:
      - name: web
        image: myapp:v2.0
        # Readiness probe gates traffic — CRITICAL for zero-downtime
        readinessProbe:
          httpGet:
            path: /health/ready
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
          failureThreshold: 3
          successThreshold: 1
        # Liveness probe restarts deadlocked pods
        livenessProbe:
          httpGet:
            path: /health/live
            port: 8080
          periodSeconds: 10
          failureThreshold: 3
        # Give in-flight requests time to complete
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 15"]
      # Must be >= preStop sleep time
      terminationGracePeriodSeconds: 30

# Trigger update
kubectl set image deployment/web-app web=myapp:v2.0 -n production

# Watch real-time rollout progress
kubectl rollout status deployment/web-app -n production

# Inspect revision history
kubectl rollout history deployment/web-app -n production

# Instant rollback to previous version
kubectl rollout undo deployment/web-app -n production

# Rollback to specific revision
kubectl rollout undo deployment/web-app --to-revision=3 -n production

# Pause mid-rollout (canary-style manual gate)
kubectl rollout pause deployment/web-app -n production
# Check metrics, error rates...
kubectl rollout resume deployment/web-app -n production

apiVersion: v1
kind: Pod
metadata:
  name: web-app
spec:
  initContainers:

  # 1. Wait for database to accept connections
  - name: wait-for-postgres
    image: busybox:1.35
    command:
    - sh
    - -c
    - |
      until nc -z postgres-service.db.svc.cluster.local 5432; do
        echo "Waiting for PostgreSQL..."
        sleep 3
      done
      echo "PostgreSQL is ready!"

  # 2. Run database migrations
  - name: run-migrations
    image: myapp:v2.0
    command: ["python", "manage.py", "migrate", "--no-input"]
    env:
    - name: DATABASE_URL
      valueFrom:
        secretKeyRef:
          name: db-secret
          key: url

  # 3. Fetch runtime config from S3
  - name: fetch-config
    image: amazon/aws-cli:latest
    command:
    - sh
    - -c
    - aws s3 cp s3://my-bucket/config/app.json /config/app.json
    volumeMounts:
    - name: config-vol
      mountPath: /config

  containers:
  - name: app
    image: myapp:v2.0
    volumeMounts:
    - name: config-vol
      mountPath: /app/config
      readOnly: true
    readinessProbe:
      httpGet:
        path: /ready
        port: 8080
      initialDelaySeconds: 5

  volumes:
  - name: config-vol
    emptyDir: {}

apiVersion: v1
kind: Pod
spec:
  containers:
  - name: app
    image: myapp:v2.0

    # STARTUP PROBE — Checked first. Disables liveness/readiness until it passes.
    # Use for apps with slow/variable startup times.
    startupProbe:
      httpGet:
        path: /health
        port: 8080
      failureThreshold: 30   # 30 * 10s = 5 minutes max startup time
      periodSeconds: 10
      # If this fails 30 times, container is killed and restarted

    # LIVENESS PROBE — Is the app alive? Kills and restarts if failing.
    # Don't call slow external dependencies here (DB, external APIs)
    livenessProbe:
      httpGet:
        path: /health/live
        port: 8080
      periodSeconds: 10
      failureThreshold: 3       # Kill after 3 consecutive failures (30s)
      timeoutSeconds: 5
      successThreshold: 1

    # READINESS PROBE — Is the app ready to serve traffic?
    # Removes pod from Service endpoints when failing.
    readinessProbe:
      httpGet:
        path: /health/ready    # Can check DB connectivity here
        port: 8080
      periodSeconds: 5
      failureThreshold: 3      # Remove from LB after 15s of failures
      successThreshold: 2      # Re-add only after 2 consecutive passes
      initialDelaySeconds: 0   # startupProbe handles initial delay

    # TCP socket probe (databases, non-HTTP services)
    # livenessProbe:
    #   tcpSocket:
    #     port: 5432
    #   periodSeconds: 10

    # Exec probe (custom health check command)
    # livenessProbe:
    #   exec:
    #     command: ["redis-cli", "ping"]
    #   periodSeconds: 10

# Stable Deployment — 9 replicas = 90% traffic
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app-stable
  namespace: production
spec:
  replicas: 9
  selector:
    matchLabels:
      app: web-app
      track: stable
  template:
    metadata:
      labels:
        app: web-app     # ← shared label
        track: stable
    spec:
      containers:
      - name: web
        image: myapp:v1.0
---
# Canary Deployment — 1 replica = 10% traffic
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app-canary
  namespace: production
spec:
  replicas: 1
  selector:
    matchLabels:
      app: web-app
      track: canary
  template:
    metadata:
      labels:
        app: web-app     # ← same shared label
        track: canary
    spec:
      containers:
      - name: web
        image: myapp:v2.0
---
# Service selects ALL pods with app=web-app
# Traffic split is proportional to replica count (9:1 = 90%:10%)
apiVersion: v1
kind: Service
metadata:
  name: web-app
spec:
  selector:
    app: web-app         # Matches BOTH stable and canary pods
  ports:
  - port: 80
    targetPort: 8080

# Step 1: Deploy canary at 10%
kubectl apply -f canary-deployment.yaml

# Step 2: Monitor error rate and latency
kubectl logs -l track=canary --tail=200 -n production
kubectl top pods -l track=canary -n production

# Step 3a: Promote — gradually increase canary, decrease stable
kubectl scale deployment/web-app-canary --replicas=3   # 30%
kubectl scale deployment/web-app-stable --replicas=7   # 70%
# ... eventually
kubectl scale deployment/web-app-canary --replicas=10  # 100%
kubectl scale deployment/web-app-stable --replicas=0

# Step 3b: Rollback if issues found
kubectl scale deployment/web-app-canary --replicas=0
kubectl delete deployment/web-app-canary

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: datadog-agent
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: datadog-agent
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app: datadog-agent
    spec:
      # Required to run on control-plane nodes too
      tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
      - key: node.kubernetes.io/not-ready
        operator: Exists
        effect: NoExecute

      # Only run on Linux nodes
      nodeSelector:
        kubernetes.io/os: linux

      containers:
      - name: agent
        image: datadog/agent:latest
        env:
        - name: DD_API_KEY
          valueFrom:
            secretKeyRef:
              name: datadog-secret
              key: api-key
        - name: DD_KUBERNETES_KUBELET_HOST
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        resources:
          requests:
            cpu: 200m
            memory: 256Mi
          limits:
            cpu: 500m
            memory: 512Mi
        volumeMounts:
        - name: dockersocket
          mountPath: /var/run/docker.sock
        - name: procdir
          mountPath: /host/proc
          readOnly: true
        - name: cgroups
          mountPath: /host/sys/fs/cgroup
          readOnly: true

      volumes:
      - name: dockersocket
        hostPath:
          path: /var/run/docker.sock
      - name: procdir
        hostPath:
          path: /proc
      - name: cgroups
        hostPath:
          path: /sys/fs/cgroup

# Parallel batch Job — process items in parallel
apiVersion: batch/v1
kind: Job
metadata:
  name: data-processor
  namespace: production
spec:
  completions: 10          # Total tasks to complete
  parallelism: 3           # Run 3 pods simultaneously
  backoffLimit: 4          # Retry up to 4 times per pod
  activeDeadlineSeconds: 3600   # Kill job after 1 hour
  ttlSecondsAfterFinished: 86400  # Auto-delete after 24h
  template:
    spec:
      restartPolicy: OnFailure   # OnFailure or Never (not Always)
      containers:
      - name: processor
        image: data-processor:v1.0
        command:
        - python
        - process.py
        - --chunk-index=$(JOB_COMPLETION_INDEX)
        env:
        - name: JOB_COMPLETION_INDEX
          valueFrom:
            fieldRef:
              fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index']
        resources:
          requests:
            cpu: 500m
            memory: 1Gi

# CronJob — nightly database cleanup at 2 AM UTC
apiVersion: batch/v1
kind: CronJob
metadata:
  name: nightly-cleanup
  namespace: production
spec:
  schedule: "0 2 * * *"          # Cron: minute hour dom month dow
  timeZone: "UTC"                 # K8s 1.27+
  concurrencyPolicy: Forbid       # Skip if previous job still running
  successfulJobsHistoryLimit: 3   # Keep last 3 successful job records
  failedJobsHistoryLimit: 3       # Keep last 3 failed job records
  startingDeadlineSeconds: 300    # If missed window, only retry within 5 min
  jobTemplate:
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 1800
      template:
        spec:
          restartPolicy: OnFailure
          serviceAccountName: cleanup-sa
          containers:
          - name: cleanup
            image: myapp:v2.0
            command: ["python", "cleanup.py", "--days=30"]
            resources:
              requests:
                cpu: 100m
                memory: 256Mi

# Manually trigger a CronJob immediately
kubectl create job --from=cronjob/nightly-cleanup manual-run-$(date +%s)

# Monitor Job progress
kubectl get jobs -n production
kubectl describe job data-processor -n production
kubectl logs -l job-name=data-processor --prefix=true

apiVersion: v1
kind: Pod
metadata:
  name: app-with-sidecars
spec:
  # Shared volume for inter-container communication
  volumes:
  - name: shared-logs
    emptyDir: {}
  - name: nginx-config
    configMap:
      name: nginx-config

  containers:
  # Main application (HTTP on localhost:8080)
  - name: app
    image: myapp:v2.0
    ports:
    - containerPort: 8080
    volumeMounts:
    - name: shared-logs
      mountPath: /var/log/app

  # Sidecar 1: NGINX reverse proxy / TLS terminator
  - name: nginx-proxy
    image: nginx:1.25-alpine
    ports:
    - containerPort: 443
    volumeMounts:
    - name: nginx-config
      mountPath: /etc/nginx/conf.d
    - name: tls-certs
      mountPath: /etc/nginx/certs
      readOnly: true

  # Sidecar 2: Log shipper
  - name: log-shipper
    image: fluent/fluent-bit:latest
    volumeMounts:
    - name: shared-logs
      mountPath: /var/log/app
      readOnly: true

  # Sidecar 3: Vault Agent (secrets injection)
  - name: vault-agent
    image: hashicorp/vault:latest
    command: ["vault", "agent", "-config=/vault/config/agent.hcl"]
    volumeMounts:
    - name: vault-config
      mountPath: /vault/config
    - name: secrets-vol
      mountPath: /vault/secrets

  initContainers:
  # Init: Set up Vault auth token before main containers start
  - name: vault-init
    image: hashicorp/vault:latest
    command: ["sh", "-c", "vault login -method=kubernetes role=my-app"]

  volumes:
  - name: tls-certs
    secret:
      secretName: app-tls-cert
  - name: vault-config
    configMap:
      name: vault-agent-config
  - name: secrets-vol
    emptyDir:
      medium: Memory  # Store secrets in RAM, not disk

apiVersion: apps/v1
kind: Deployment
spec:
  template:
    spec:
      # Must be >= preStop sleep + app shutdown time
      terminationGracePeriodSeconds: 60

      containers:
      - name: app
        image: myapp:v2.0

        lifecycle:
          preStop:
            exec:
              # Sleep bridges the gap between endpoint removal
              # and actual traffic drain (~5-15s is typical)
              command: ["/bin/sh", "-c", "sleep 15"]

          # OR: call a graceful shutdown endpoint
          # preStop:
          #   httpGet:
          #     path: /shutdown
          #     port: 8080

        # Your app should handle SIGTERM gracefully:
        # - Stop accepting new connections
        # - Finish processing in-flight requests
        # - Close DB connections cleanly
        # - Exit with code 0

# Example: Python app handling SIGTERM
import signal, sys

def handle_sigterm(signum, frame):
    print("Received SIGTERM — finishing in-flight requests...")
    server.shutdown()  # Stop accepting new requests
    # Wait for in-flight requests to complete
    sys.exit(0)

signal.signal(signal.SIGTERM, handle_sigterm)

apiVersion: v1
kind: Pod
metadata:
  name: web-with-exporter
spec:
  volumes:
  # Shared socket directory between containers
  - name: socket-dir
    emptyDir: {}
  # Shared log directory
  - name: log-dir
    emptyDir: {}

  containers:
  # Main app: writes to /tmp/app.sock and /var/log/app/
  - name: web-server
    image: myapp:v2.0
    volumeMounts:
    - name: socket-dir
      mountPath: /tmp
    - name: log-dir
      mountPath: /var/log/app

  # Exporter: reads from the same socket/logs
  - name: metrics-exporter
    image: prom/statsd-exporter:latest
    args:
    - --statsd.listen-unixgram=/tmp/app.sock
    volumeMounts:
    - name: socket-dir
      mountPath: /tmp   # Same path — shares the socket file
    ports:
    - containerPort: 9102
      name: metrics
    readinessProbe:
      httpGet:
        path: /metrics
        port: 9102

  # Log rotator sidecar
  - name: log-rotator
    image: busybox:1.35
    command: ["/bin/sh", "-c"]
    args:
    - |
      while true; do
        find /var/log/app -name "*.log" -mtime +7 -delete
        sleep 3600
      done
    volumeMounts:
    - name: log-dir
      mountPath: /var/log/app

# ClusterIP (default) — cluster-internal only
apiVersion: v1
kind: Service
metadata:
  name: backend-api
spec:
  type: ClusterIP
  selector:
    app: backend
  ports:
  - port: 8080
    targetPort: 8080
  # DNS: backend-api.namespace.svc.cluster.local:8080

# NodePort — expose on every node's IP:port
apiVersion: v1
kind: Service
metadata:
  name: web-nodeport
spec:
  type: NodePort
  selector:
    app: web
  ports:
  - port: 80
    targetPort: 8080
    nodePort: 30080    # Valid range: 30000-32767
  # Access: <any-node-ip>:30080

# LoadBalancer — provisions cloud load balancer (AWS NLB/ALB)
apiVersion: v1
kind: Service
metadata:
  name: web-lb
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
    service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
spec:
  type: LoadBalancer
  selector:
    app: web
  ports:
  - port: 443
    targetPort: 8080
    protocol: TCP
  # AWS provisions an NLB; service gets external DNS/IP

# Headless — no ClusterIP; DNS returns individual pod IPs
apiVersion: v1
kind: Service
metadata:
  name: postgres-headless
spec:
  clusterIP: None
  selector:
    app: postgres
  ports:
  - port: 5432
  # DNS: postgres-0.postgres-headless.ns.svc.cluster.local → pod-0 IP
  #       postgres-1.postgres-headless.ns.svc.cluster.local → pod-1 IP

# Install NGINX Ingress Controller
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm install ingress-nginx ingress-nginx/ingress-nginx \
  --namespace ingress-nginx --create-namespace \
  --set controller.metrics.enabled=true

# Install cert-manager
helm repo add jetstack https://charts.jetstack.io
helm install cert-manager jetstack/cert-manager \
  --namespace cert-manager --create-namespace \
  --set installCRDs=true

# ClusterIssuer — Let's Encrypt certificate authority
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: admin@example.com
    privateKeySecretRef:
      name: letsencrypt-prod-key
    solvers:
    - http01:
        ingress:
          class: nginx

# Ingress — path-based routing + automatic TLS
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: api-ingress
  namespace: production
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "60"
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
    # Rate limiting
    nginx.ingress.kubernetes.io/limit-rps: "100"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - api.example.com
    secretName: api-example-com-tls   # cert-manager creates this
  rules:
  - host: api.example.com
    http:
      paths:
      - path: /users
        pathType: Prefix
        backend:
          service:
            name: users-service
            port:
              number: 8080
      - path: /orders
        pathType: Prefix
        backend:
          service:
            name: orders-service
            port:
              number: 8080
      - path: /
        pathType: Prefix
        backend:
          service:
            name: frontend-service
            port:
              number: 3000

# Monitor certificate issuance
kubectl get certificate -n production
kubectl describe certificate api-example-com-tls -n production
kubectl get certificaterequest -n production

# Launch a debug pod with DNS tools
kubectl run dns-debug \
  --image=registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3 \
  --restart=Never -it -- bash

# Inside the pod:
# Check resolv.conf
cat /etc/resolv.conf
# nameserver 10.96.0.10         ← CoreDNS ClusterIP
# search default.svc.cluster.local svc.cluster.local cluster.local
# options ndots:5

# Test service DNS
nslookup kubernetes.default.svc.cluster.local
nslookup my-service.production.svc.cluster.local

# Test external DNS
nslookup google.com

# Debug CoreDNS
kubectl get pods -n kube-system -l k8s-app=kube-dns
kubectl logs -n kube-system -l k8s-app=kube-dns --prefix

# Check CoreDNS config
kubectl get configmap coredns -n kube-system -o yaml

# Common CoreDNS issues:
# 1. CoreDNS pods OOMKilled → increase memory limits
# 2. ndots:5 causing slow resolution → external lookups try 6 DNS queries
#    Fix: set ndots:1 in pod dnsConfig for external-heavy workloads
# 3. DNS cache poisoning → use separate upstream resolvers

# Override DNS per pod
apiVersion: v1
kind: Pod
spec:
  dnsConfig:
    options:
    - name: ndots
      value: "1"    # Reduces unnecessary search path queries
    nameservers:
    - 8.8.8.8       # Additional custom nameservers

# Step 1: Default-deny ALL ingress AND egress in the namespace
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: production
spec:
  podSelector: {}     # Applies to ALL pods
  policyTypes:
  - Ingress
  - Egress

# Step 2: Allow frontend → API (port 8080 only)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-frontend-to-api
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: api-server
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend
    ports:
    - protocol: TCP
      port: 8080

# Step 3: Allow API → PostgreSQL (port 5432 only)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-api-to-postgres
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: postgres
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: api-server
    ports:
    - protocol: TCP
      port: 5432

# Step 4: Allow all pods to query CoreDNS (essential!)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns-egress
  namespace: production
spec:
  podSelector: {}
  policyTypes:
  - Egress
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53

# Step 5: Allow API egress to external services
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-api-external-egress
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: api-server
  policyTypes:
  - Egress
  egress:
  - ports:
    - protocol: TCP
      port: 443    # HTTPS to external APIs
  - to:
    - podSelector:
        matchLabels:
          app: postgres
    ports:
    - protocol: TCP
      port: 5432

# ExternalName — DNS alias to external service
# App uses: postgres-prod.production.svc.cluster.local
# Resolves to: prod-db.abc123.us-east-1.rds.amazonaws.com
apiVersion: v1
kind: Service
metadata:
  name: postgres-prod
  namespace: production
spec:
  type: ExternalName
  externalName: prod-db.abc123.us-east-1.rds.amazonaws.com
  # No selector — no pods. Pure DNS CNAME alias.

# Endpoints + Service — point to external IP directly
# Use when external service doesn't have a hostname
apiVersion: v1
kind: Service
metadata:
  name: legacy-payment-api
  namespace: production
spec:
  ports:
  - port: 443
    targetPort: 443
---
apiVersion: v1
kind: Endpoints
metadata:
  name: legacy-payment-api   # Must match Service name
subsets:
- addresses:
  - ip: 10.20.30.40          # External server IP
  - ip: 10.20.30.41
  ports:
  - port: 443

# ServiceEntry (Istio) — fine-grained external service control
apiVersion: networking.istio.io/v1beta1
kind: ServiceEntry
metadata:
  name: stripe-api
spec:
  hosts:
  - api.stripe.com
  ports:
  - number: 443
    name: https
    protocol: HTTPS
  resolution: DNS
  location: MESH_EXTERNAL

# Check current kube-proxy mode
kubectl get configmap kube-proxy -n kube-system -o yaml | grep mode

# Switch to IPVS mode
kubectl edit configmap kube-proxy -n kube-system
# Set: mode: "ipvs"

# Restart kube-proxy pods to apply
kubectl rollout restart daemonset/kube-proxy -n kube-system

# Verify IPVS rules
ipvsadm -Ln   # Run on a node

# Cilium — can replace kube-proxy entirely (eBPF)
# No iptables, no kube-proxy, direct eBPF kernel datapath
helm install cilium cilium/cilium \
  --namespace kube-system \
  --set kubeProxyReplacement=true \
  --set k8sServiceHost=<api-server-ip>

# Option 1: Istio service mesh — automatic mTLS
helm repo add istio https://istio-release.storage.googleapis.com/charts
helm install istio-base istio/base -n istio-system --create-namespace
helm install istiod istio/istiod -n istio-system

# Label namespace for automatic sidecar injection
kubectl label namespace production istio-injection=enabled

# Enforce strict mTLS across namespace

# Istio PeerAuthentication — enforce mTLS
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: enforce-mtls
  namespace: production
spec:
  mtls:
    mode: STRICT    # STRICT: only mTLS, no plaintext
    # PERMISSIVE: both mTLS and plaintext (migration mode)
    # DISABLE: no mTLS

# Istio AuthorizationPolicy — service-level access control
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: api-server-policy
  namespace: production
spec:
  selector:
    matchLabels:
      app: api-server
  action: ALLOW
  rules:
  - from:
    - source:
        principals:
        # Only allow calls from frontend service account
        - "cluster.local/ns/production/sa/frontend-sa"
    to:
    - operation:
        methods: ["GET", "POST"]
        paths: ["/api/*"]

# Verify mTLS is working
istioctl authn tls-check api-server-pod-xxx.production

# View certificate details
istioctl proxy-config secret api-server-pod-xxx.production

# Topology Aware Routing (K8s 1.27+)
apiVersion: v1
kind: Service
metadata:
  name: web-app
  annotations:
    service.kubernetes.io/topology-mode: "Auto"
    # "Auto" — route to same-zone endpoints when possible
    # Fallback to cross-zone if no local endpoints are healthy
spec:
  selector:
    app: web-app
  ports:
  - port: 80
    targetPort: 8080

# TrafficPolicy in Istio (more fine-grained control)
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: web-app
  namespace: production
spec:
  host: web-app
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100
      http:
        h2UpgradePolicy: UPGRADE
        idleTimeout: 10s
    loadBalancer:
      localityLbSetting:
        enabled: true
        distribute:
        - from: "us-east-1/us-east-1a/*"
          to:
            "us-east-1/us-east-1a/*": 80   # 80% same-AZ
            "us-east-1/us-east-1b/*": 20   # 20% different AZ
    outlierDetection:
      consecutive5xxErrors: 5
      interval: 30s
      baseEjectionTime: 30s

# StorageClass — tells K8s how to provision storage
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast-ssd
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: ebs.csi.aws.com
parameters:
  type: gp3
  iops: "3000"
  throughput: "125"
  encrypted: "true"
reclaimPolicy: Retain                    # Keep EBS volume when PVC deleted
allowVolumeExpansion: true               # Allow PVC resize
volumeBindingMode: WaitForFirstConsumer  # Create volume in same AZ as pod