From ea551e5278b8ee6c2c13f51682572a57d3abf0eb Mon Sep 17 00:00:00 2001
From: Carl Tashian <carl@smallstep.com>
Date: Wed, 2 Feb 2022 15:54:16 -0800
Subject: [PATCH] Lots of updates on cert renewal

---
 docs/04-certificate-authority.md              |  13 +-
 ...08-bootstrapping-kubernetes-controllers.md |   3 +
 docs/13-certificate-renewal.md                | 152 +++++++++++++++++-
 3 files changed, 154 insertions(+), 14 deletions(-)

diff --git a/docs/04-certificate-authority.md b/docs/04-certificate-authority.md
index 5844642..99e7673 100644
--- a/docs/04-certificate-authority.md
+++ b/docs/04-certificate-authority.md
@@ -18,9 +18,7 @@ Download the `step` client and `step-ca` server binaries, and the `jq` command:
 {
 wget -q --show-progress --https-only --timestamping \
   "https://dl.step.sm/gh-release/certificates/gh-release-header/v0.18.0/step-ca_linux_0.18.0_amd64.tar.gz" \
-  "https://dl.step.sm/gh-release/cli/gh-release-header/v0.18.0/step_linux_0.18.0_amd64.tar.gz" \
-  "https://raw.githubusercontent.com/smallstep/cli/master/systemd/cert-renewer%40.service" \
-  "https://raw.githubusercontent.com/smallstep/cli/master/systemd/cert-renewer%40.timer"
+  "https://dl.step.sm/gh-release/cli/gh-release-header/v0.18.0/step_linux_0.18.0_amd64.tar.gz"
 sudo apt update
 sudo apt install -y jq
 }
@@ -31,9 +29,9 @@ Install the binaries:
 ```
 {
 tar -xvf step-ca_linux_0.18.0_amd64.tar.gz
-sudo mv step-ca_0.18.0/bin/* /usr/local/bin/
+sudo mv step-ca_0.18.0/bin/step-ca /usr/local/bin/
 tar -xvf step_linux_0.18.0_amd64.tar.gz
-sudo mv step_0.18.0/bin/* /usr/local/bin/
+sudo mv step_0.18.0/bin/step /usr/local/bin/
 }
 ```
 
@@ -441,7 +439,10 @@ Copy the appropriate certificates and private keys to each controller instance:
 ```
 for instance in controller-0 controller-1 controller-2; do
   gcloud compute scp ca.pem kubernetes-key.pem kubernetes.pem \
-    service-account-key.pem service-account.pem ${instance}:~/
+    service-account-key.pem service-account.pem \
+    kube-controller-manager-key.pem kube-controller-manager.pem \
+    kube-proxy-key.pem kube-proxy.pem kube-scheduler-key.pem \
+    kube-scheduler.pem ${instance}:~/
 done
 ```
 
diff --git a/docs/08-bootstrapping-kubernetes-controllers.md b/docs/08-bootstrapping-kubernetes-controllers.md
index 7bf1384..d3d90fe 100644
--- a/docs/08-bootstrapping-kubernetes-controllers.md
+++ b/docs/08-bootstrapping-kubernetes-controllers.md
@@ -51,6 +51,9 @@ Install the Kubernetes binaries:
 
   sudo mv ca.pem kubernetes-key.pem kubernetes.pem \
     service-account-key.pem service-account.pem \
+    kube-proxy.pem kube-proxy-key.pem \
+    kube-controller-manager.pem kube-controller-manager-key.pem \
+    kube-scheduler.pem kube-scheduler-key.pem \
     encryption-config.yaml /var/lib/kubernetes/
 }
 ```
diff --git a/docs/13-certificate-renewal.md b/docs/13-certificate-renewal.md
index d0c77e0..338892b 100644
--- a/docs/13-certificate-renewal.md
+++ b/docs/13-certificate-renewal.md
@@ -21,9 +21,7 @@ Install the binary and renewal utility files:
 
 ```
 tar -xvf step_linux_0.18.0_amd64.tar.gz
-sudo mv step_0.18.0/bin/* /usr/local/bin/
-sudo mv cert-renewer@.service /etc/systemd/system
-sudo mv cert-renewer@.timer /etc/systemd/system
+sudo mv step_0.18.0/bin/step /usr/local/bin/
 sudo systemctl daemon-reload
 ```
 
@@ -48,6 +46,66 @@ The root certificate has been saved in /root/.step/certs/root_ca.crt.
 The authority configuration has been saved in /root/.step/config/defaults.json.
 ```
 
+## Set up the certificate renewal timer
+
+We'll use a systemd timer to renew certificates when they are 2/3rds of the way through their validity period.
+
+Install the systemd certificate renewal service and timer.
+
+```
+cat << EOF | sudo tee /etc/systemd/system/cert-renewer@.service
+[Unit]
+Description=Certificate renewer for %I
+After=network-online.target
+Documentation=https://smallstep.com/docs/step-ca/certificate-authority-server-production
+StartLimitIntervalSec=0
+
+[Service]
+Type=oneshot
+User=root
+
+Environment=STEPPATH=/etc/step-ca \
+            CERT_LOCATION=/etc/step/certs/%i.crt \
+            KEY_LOCATION=/etc/step/certs/%i.key
+
+; ExecCondition checks if the certificate is ready for renewal,
+; based on the exit status of the command.
+; (In systemd <242, you can use ExecStartPre= here.)
+ExecCondition=/usr/local/bin/step certificate needs-renewal ${CERT_LOCATION}
+
+; ExecStart renews the certificate, if ExecStartPre was successful.
+ExecStart=/usr/local/bin/step ca renew --force ${CERT_LOCATION} ${KEY_LOCATION}
+
+[Install]
+WantedBy=multi-user.target
+EOF
+```
+
+Install the timer:
+
+```
+cat << EOF | sudo tee /etc/systemd/system/cert-renewer@.timer
+[Unit]
+Description=Certificate renewal timer for %I
+Documentation=https://smallstep.com/docs/step-ca/certificate-authority-server-production
+
+[Timer]
+Persistent=true
+
+; Run the timer unit every 5 minutes.
+OnCalendar=*:1/5
+
+; Always run the timer on time.
+AccuracySec=1us
+
+; Add jitter to prevent a "thundering hurd" of simultaneous certificate renewals.
+RandomizedDelaySec=5m
+
+[Install]
+WantedBy=timers.target
+EOF
+```
+
 ## Configure certificate renewal for etcd
 
 Create and start a certificate renewal timer for etcd:
@@ -59,20 +117,98 @@ cat <<EOF | sudo tee /etc/systemd/system/cert-renewer@etcd.service.d/override.co
 Environment=STEPPATH=/root/.step \\
             CERT_LOCATION=/etc/etcd/kubernetes.pem \\
             KEY_LOCATION=/etc/etcd/kubernetes-key.pem
-
-; Don't try to restart etcd.service; etcd will read
-; certificates from disk on every new request.
-ExecStartPost=
 EOF
 sudo systemctl daemon-reload
 sudo systemctl enable --now cert-renewer@etcd.timer
 ```
 
-## Configure certificate renewal for kube-controller-manager
+## Configure certificate renewal for `kube-controller-manager`
 
+Create and start a certificate renewal timer for `kube-controller-manager`. This one will use `kubectl` to embed the renewed certificate and key into the kubeconfig file before restarting the controller manager. Run:
 
+```
+sudo mkdir /etc/systemd/system/cert-renewer@kube-controller-manager.service.d
+cat <<EOF | sudo tee /etc/systemd/system/cert-renewer@kube-controller-manager.service.d/override.conf
+[Service]
+Environment=STEPPATH=/root/.step \\
+            CERT_LOCATION=/var/lib/kubernetes/kube-controller-manager.pem
+ \\
+            KEY_LOCATION=/var/lib/kubernetes/kube-controller-manager-key.pem
+
+ExecStartPost=kubectl config set-credentials system:kube-controller-manager \\
+    --client-certificate=\${CERT_LOCATION} \\
+    --client-key=\${KEY_LOCATION} \\
+    --embed-certs=true \\
+    --kubeconfig=/var/lib/kubernetes/kube-controller-manager.kubeconfig
+
+ExecStartPost=systemctl restart kube-controller-manager.service
+EOF
+sudo systemctl daemon-reload
+sudo systemctl enable --now cert-renewer@kube-controller-manager.timer
+```
 
 ## Configure certificate renewal for kube-scheduler
 
+Create and start a certificate renewal timer for `kube-scheduler`:
+
+```
+sudo mkdir /etc/systemd/system/cert-renewer@kube-scheduler.service.d
+cat <<EOF | sudo tee /etc/systemd/system/cert-renewer@kube-scheduler.service.d/override.conf
+[Service]
+Environment=STEPPATH=/root/.step \\
+            CERT_LOCATION=/var/lib/kubernetes/kube-scheduler.pem
+ \\
+            KEY_LOCATION=/var/lib/kubernetes/kube-scheduler-key.pem
+
+ExecStartPost=kubectl config set-credentials system:kube-scheduler \\
+    --client-certificate=\${CERT_LOCATION} \\
+    --client-key=\${KEY_LOCATION} \\
+    --embed-certs=true \\
+    --kubeconfig=/var/lib/kubernetes/kube-scheduler.kubeconfig
+
+ExecStartPost=systemctl restart kube-scheduler.service
+EOF
+sudo systemctl daemon-reload
+sudo systemctl enable --now cert-renewer@kube-scheduler.timer
+```
+
+
+## Configure certificate renewal for kube-apiserver
+
+Create and start a certificate renewal timer for `kube-apiserver`:
+
+```
+sudo mkdir /etc/systemd/system/cert-renewer@kube-apiserver.service.d
+cat <<EOF | sudo tee /etc/systemd/system/cert-renewer@kube-apiserver.service.d/override.conf
+[Service]
+Environment=STEPPATH=/root/.step \\
+            CERT_LOCATION=/var/lib/kubernetes/kubernetes.pem  \\
+            KEY_LOCATION=/var/lib/kubernetes/kubernetes-key.pem 
+
+ExecStartPost=systemctl restart kube-apiserver.service
+EOF
+sudo systemctl daemon-reload
+sudo systemctl enable --now cert-renewer@kube-apiserver.timer
+```
+
+
+## Configure service account certificate renewal timer
+
+```
+sudo mkdir /etc/systemd/system/cert-renewer@kube-service-account.service.d
+cat <<EOF | sudo tee /etc/systemd/system/cert-renewer@kube-service-account.service.d/override.conf
+[Service]
+Environment=STEPPATH=/root/.step \\
+            CERT_LOCATION=/var/lib/kubernetes/service-account.pem
+ \\
+            KEY_LOCATION=/var/lib/kubernetes/service-account-key.pem
+
+; Restart services that use the service account certificate or key
+ExecStartPost=systemctl restart kube-controller-manager.service
+ExecStartPost=systemctl restart kube-apiserver.service
+sudo systemctl daemon-reload
+sudo systemctl enable --now cert-renewer@kube-service-account.timer
+EOF
+```
 
 > Remember to run the above commands on each controller node: `controller-0`, `controller-1`, and `controller-2`.