mirrors/kubernetes-the-hard-way
1
0
Fork 0
mirror of https://github.com/kelseyhightower/kubernetes-the-hard-way.git synced 2025-12-15 17:28:58 +03:00
Code Issues Projects Releases Wiki Activity

Refresh and add Apple Silicon (#338)

Browse Source 
* Delete CKA stuff.
It's covered in CKA repo

* Rename nodes

* Cluster up again

* Update issue template

* Update README

* Begin rearranging docs

* Update links

* Initial mac instructions

* iterm2 image

* update ssh-copy-id to be cross platform

* remove vagrant specific

* Apple scripts WIP

* Add var for architecture

* order input files

* Apple build working!

* auto-locate docs

* install sshpass

* Set execute bit

* apple done!

* install sshpass

* edits

* Corrections

* kube version output

* Adjustments

* Adjustments
This commit is contained in:
Alistair Mackay
2024-03-18 05:16:56 +00:00
committed by GitHub
parent 645b296cb6
commit 65b5a11784
81 changed files with 1400 additions and 787 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
apple-silicon/delete-virtual-machines.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -eo pipefail
specs=/tmp/vm-specs
cat <<EOF > $specs
controlplane01,2,2048M,10G
controlplane02,2,2048M,5G
loadbalancer,1,512M,5G
node01,2,2048M,5G
node02,2,2048M,5G
EOF
for spec in $(cat $specs)
do
n=$(cut -d ',' -f 1 <<< $spec)
multipass stop $n
multipass delete $n
done
multipass purge
echo
echo "You should now remove all the following lines from /var/db/dhcpd_leases"
echo
cat /var/db/dhcpd_leases | egrep -A 5 -B 1 '(controlplane|node|loadbalancer)'
echo
cat <<EOF
Use the following command to do this
sudo vi /var/db/dhcpd_leases
EOF

104
apple-silicon/deploy-virtual-machines.sh Executable file
View File

@@ -0,0 +1,104 @@
#!/usr/bin/env bash
# When VMs are deleted, IPs remain allocated in dhcpdb
# IP reclaim: https://discourse.ubuntu.com/t/is-it-possible-to-either-specify-an-ip-address-on-launch-or-reset-the-next-ip-address-to-be-used/30316
ARG=$1
set -euo pipefail
RED="\033[1;31m"
YELLOW="\033[1;33m"
GREEN="\033[1;32m"
BLUE="\033[1;34m"
NC="\033[0m"
echo -e "${BLUE}Checking system compatibility${NC}"
MEM_GB=$(( $(sysctl hw.memsize | cut -d ' ' -f 2) / 1073741824 ))
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/scripts
if [ $MEM_GB -lt 12 ]
then
echo -e "${RED}System RAM is ${MEM_GB}GB. This is insufficient to deploy a working cluster.${NC}"
exit 1
fi
if ! command -v multipass > /dev/null
then
echo -e "${RED}Cannot find multipass. Did you install it as per the instructions?${NC}"
exit 1
fi
if ! command -v jq > /dev/null
then
echo -e "${RED}Cannot find jq. Did you install it as per the instructions?${NC}"
exit 1
fi
specs=/tmp/vm-specs
cat <<EOF > $specs
controlplane01,2,2048M,10G
controlplane02,2,2048M,5G
loadbalancer,1,512M,5G
node01,2,2048M,5G
node02,2,2048M,5G
EOF
echo -e "${GREEN}System OK!${NC}"
# If the nodes are running, reset them
for spec in $(cat $specs)
do
node=$(cut -d ',' -f 1 <<< $spec)
if multipass list --format json | jq -r '.list[].name' | grep $node > /dev/null
then
echo -n -e $RED
read -p "VMs are running. Delete and rebuild them (y/n)? " ans
echo -n -e $NC
[ "$ans" != 'y' ] && exit 1
break
fi
done
# Boot the nodes
for spec in $(cat $specs)
do
node=$(cut -d ',' -f 1 <<< $spec)
cpus=$(cut -d ',' -f 2 <<< $spec)
ram=$(cut -d ',' -f 3 <<< $spec)
disk=$(cut -d ',' -f 4 <<< $spec)
if multipass list --format json | jq -r '.list[].name' | grep $(cut -d ',' -f 1 <<< $node) > /dev/null
then
echo -e "${YELLOW}Deleting $node${NC}"
multipass delete $node
multipass purge
fi
echo -e "${BLUE}Launching ${node}${NC}"
multipass launch --disk $disk --memory $ram --cpus $cpus --name $node jammy
echo -e "${GREEN}$node booted!${NC}"
done
# Create hostfile entries
echo -e "${BLUE}Provisioning...${NC}"
hostentries=/tmp/hostentries
[ -f $hostentries ] && rm -f $hostentries
for spec in $(cat $specs)
do
node=$(cut -d ',' -f 1 <<< $spec)
ip=$(multipass info $node --format json | jq -r 'first( .info[] | .ipv4[0] )')
echo "$ip $node" >> $hostentries
done
for spec in $(cat $specs)
do
node=$(cut -d ',' -f 1 <<< $spec)
multipass transfer $hostentries $node:/tmp/
multipass transfer $SCRIPT_DIR/01-setup-hosts.sh $node:/tmp/
multipass transfer $SCRIPT_DIR/cert_verify.sh $node:/home/ubuntu/
multipass exec $node -- /tmp/01-setup-hosts.sh
done
echo -e "${GREEN}Done!${NC}"

64
apple-silicon/docs/01-prerequisites.md Normal file
View File

@@ -0,0 +1,64 @@
# Prerequisites
## Hardware Requirements
This lab provisions 5 VMs on your workstation. That's a lot of compute resource!
* Apple Silicon System (M1/M2/M3 etc)
* Minimum 16GB RAM.<br/>Bear in mind that the unified memory architecture of Apple Silicon Macs means that the whole of the quoted memory is not available for software - some of it is used for the display, more if you have external displays.<br/>This rules out 8GB models - sorry.
* Pro or Max CPU recommended for running the e2e-tests at the end of this lab.
## Required Software
You'll need to install the following first.
* Multipass - https://multipass.run/install. Follow the instructions to install it and check it is working properly. You should be able to successfully create a test Ubuntu VM following their instructions. Delete the test VM when you're done.
* JQ - https://github.com/stedolan/jq/wiki/Installation#macos
Additionally
* Your account on your Mac must have admin privilege and be able to use `sudo`
Clone this repo down to your Mac. Open your Mac's terminal application. All commands in this guide are executed from the terminal.
```bash
mkdir ~/kodekloud
cd ~/kodekloud
git clone https://github.com/mmumshad/kubernetes-the-hard-way.git
cd kubernetes-the-hard-way/apple-silicon
```
## Virtual Machine Network
Due to how the virtualization works, the networking for each VM requires two network adapters; one used by Multipass and one used by everything else. Kubernetes components may by default bind to the Multipass adapter, which is *not* what we want, therefore we have pre-set an environment variable `PRIMARY_IP` on all VMs which is the IP address that Kubernetes components should be using. In the coming labs you will see this environment variable being used to ensure Kubernetes components bind to the correct network interface.
`PRIMARY_IP` is defined as the IP address of the network interface on the node that is connected to the network having the default gateway, and is the interface that a node will use to talk to the other nodes.
### NAT Networking
In NAT configuration, the network on which the VMs run is isolated from your broadband router's network by a NAT gateway managed by the hypervisor. This means that VMs can see out (and connect to Internet), but you can't see in (i.e. use browser to connect to NodePorts). It is currently not possible to set up port forwarding rules in Multipass to facilitate this.
The network used by the VMs is chosen by Multipass.
It is *recommended* that you leave the pod and service networks as the defaults. If you change them then you will also need to edit the Weave networking manifests to accommodate your change.
If you do decide to change any of these, please treat as personal preference and do not raise a pull request.
## Running Commands in Parallel with iterm2
[iterm2](https://iterm2.com/) which is a popular replacement for the standard Mac terminal application can be used to run the same commands on multiple compute instances at the same time. Some labs in this tutorial require running the same commands on multiple compute instances for instance installing the Kubernetes software. In those cases you may consider using iterm2 and splitting a window into multiple panes with *Broadcast input to all panes* enabled to speed up the provisioning process.
*The use of iterm2 is optional and not required to complete this tutorial*.
![titerm2 screenshot](../../images//iterm2-broadcast.png)
To set up as per the image above, do the following in iterm2
1. Right click and select split pane horizontally
1. In each pane, connect to a different node with `Multipass shell`
1. From the `Session` menu at the top, toggle `Broadcast` -> `Broadcast input to all panes` (or press `ALT`-`CMD`-`I`). The small icon at the top right of each pane indicates broadcast mode is enabled.
Input typed or passed to one command prompt will be echoed to the others. Remember to turn off broadcast when you have finished a section that applies to multiple nodes.
Next: [Compute Resources](02-compute-resources.md)

60
apple-silicon/docs/02-compute-resources.md Normal file
View File

@@ -0,0 +1,60 @@
# Compute Resources
Because we cannot use VirtualBox and are instead using Multipass, [a script is provided](./deploy-virtual-machines.sh) to create the three VMs.
1. Run the VM deploy script from your Mac terminal application
```bash
./deploy-virtual-machines.sh
```
2. Verify you can connect to all VMs:
```bash
multipass shell controlplane01
```
You should see a command prompt like `ubuntu@controlplane01:~$`
Type the following to return to the Mac terminal
```bash
exit
```
Do this for the other controlplane, both nodes and loadbalancer.
# Deleting the Virtual Machines
When you have finished with your cluster and want to reclaim the resources, perform the following steps
1. Exit from all your VM sessions
1. Run the [delete script](../delete-virtual-machines.sh) from your Mac terminal application
```bash
./delete-virtual-machines.sh
````
1. Clean stale DHCP leases. Multipass does not do this automatically and if you do not do it yourself you will eventually run out of IP addresses on the multipass VM network.
1. Edit the following
```bash
sudo vi /var/db/dhcpd_leases
```
1. Remove all blocks that look like this, specifically those with `name` like `controlplane`, `node` or `loadbalancer`
```text
{
name=controlplane01
ip_address=192.168.64.4
hw_address=1,52:54:0:78:4d:ff
identifier=1,52:54:0:78:4d:ff
lease=0x65dc3134
}
```
1. Save the file and exit
Next: [Client tools](../../docs/03-client-tools.md)<br>
Prev: [Prerequisites](./01-prerequisites.md)

27
apple-silicon/scripts/01-setup-hosts.sh Executable file
View File

@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Set hostfile entries
sudo sed -i "/$(hostname)/d" /etc/hosts
cat /tmp/hostentries | sudo tee -a /etc/hosts &> /dev/null
# Export internal IP of primary NIC as an environment variable
echo "PRIMARY_IP=$(ip route | grep default | awk '{ print $9 }')" | sudo tee -a /etc/environment > /dev/null
# Export architecture as environment variable to download correct versions of software
echo "ARCH=arm64" | sudo tee -a /etc/environment > /dev/null
# Enable password auth in sshd so we can use ssh-copy-id
# Enable password auth in sshd so we can use ssh-copy-id
sudo sed -i --regexp-extended 's/#?PasswordAuthentication (yes|no)/PasswordAuthentication yes/' /etc/ssh/sshd_config
sudo sed -i --regexp-extended 's/#?Include \/etc\/ssh\/sshd_config.d\/\*.conf/#Include \/etc\/ssh\/sshd_config.d\/\*.conf/' /etc/ssh/sshd_config
sudo sed -i 's/KbdInteractiveAuthentication no/KbdInteractiveAuthentication yes/' /etc/ssh/sshd_config
sudo systemctl restart sshd
if [ "$(hostname)" = "controlplane01" ]
then
sh -c 'sudo apt update' &> /dev/null
sh -c 'sudo apt-get install -y sshpass' &> /dev/null
fi
# Set password for ubuntu user (it's something random by default)
echo 'ubuntu:ubuntu' | sudo chpasswd

23
apple-silicon/scripts/02-setup-kernel.sh Executable file
View File

@@ -0,0 +1,23 @@
#!/usr/bin/env bash
# Step 2 - Set up Operating System Prerequisites
# Load required kernel modules
sudo modprobe overlay
sudo modprobe br_netfilter
# Persist modules between restarts
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
# Set required networking parameters
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
# Apply sysctl params without reboot
sudo sysctl --system

578
apple-silicon/scripts/cert_verify.sh Executable file
View File

@@ -0,0 +1,578 @@
#!/bin/bash
set -e
#set -x
# Green & Red marking for Success and Failed messages
SUCCESS='\033[0;32m'
FAILED='\033[0;31;1m'
NC='\033[0m'
# IP addresses
PRIMARY_IP=$(ip route | grep default | awk '{ print $9 }')
CONTROL01=$(dig +short controlplane01)
CONTROL02=$(dig +short controlplane02)
NODE01=$(dig +short node01)
NODE02=$(dig +short node02)
LOADBALANCER=$(dig +short loadbalancer)
LOCALHOST="127.0.0.1"
# All Cert Location
# ca certificate location
CACERT=ca.crt
CAKEY=ca.key
# Kube controller manager certificate location
KCMCERT=kube-controller-manager.crt
KCMKEY=kube-controller-manager.key
# Kube proxy certificate location
KPCERT=kube-proxy.crt
KPKEY=kube-proxy.key
# Kube scheduler certificate location
KSCERT=kube-scheduler.crt
KSKEY=kube-scheduler.key
# Kube api certificate location
APICERT=kube-apiserver.crt
APIKEY=kube-apiserver.key
# ETCD certificate location
ETCDCERT=etcd-server.crt
ETCDKEY=etcd-server.key
# Service account certificate location
SACERT=service-account.crt
SAKEY=service-account.key
# All kubeconfig locations
# kubeproxy.kubeconfig location
KPKUBECONFIG=kube-proxy.kubeconfig
# kube-controller-manager.kubeconfig location
KCMKUBECONFIG=kube-controller-manager.kubeconfig
# kube-scheduler.kubeconfig location
KSKUBECONFIG=kube-scheduler.kubeconfig
# admin.kubeconfig location
ADMINKUBECONFIG=admin.kubeconfig
# All systemd service locations
# etcd systemd service
SYSTEMD_ETCD_FILE=/etc/systemd/system/etcd.service
# kub-api systemd service
SYSTEMD_API_FILE=/etc/systemd/system/kube-apiserver.service
# kube-controller-manager systemd service
SYSTEMD_KCM_FILE=/etc/systemd/system/kube-controller-manager.service
# kube-scheduler systemd service
SYSTEMD_KS_FILE=/etc/systemd/system/kube-scheduler.service
### WORKER NODES ###
# Worker-1 cert details
NODE01_CERT=/var/lib/kubelet/node01.crt
NODE01_KEY=/var/lib/kubelet/node01.key
# Worker-1 kubeconfig location
NODE01_KUBECONFIG=/var/lib/kubelet/kubeconfig
# Worker-1 kubelet config location
NODE01_KUBELET=/var/lib/kubelet/kubelet-config.yaml
# Systemd node01 kubelet location
SYSTEMD_NODE01_KUBELET=/etc/systemd/system/kubelet.service
# kube-proxy node01 location
NODE01_KP_KUBECONFIG=/var/lib/kube-proxy/kubeconfig
SYSTEMD_NODE01_KP=/etc/systemd/system/kube-proxy.service
# Function - Master node #
check_cert_and_key()
{
local name=$1
local subject=$2
local issuer=$3
local nokey=
local cert="${CERT_LOCATION}/$1.crt"
local key="${CERT_LOCATION}/$1.key"
if [ -z $cert -o -z $key ]
then
printf "${FAILED}cert and/or key not present in ${CERT_LOCATION}. Perhaps you missed a copy step\n${NC}"
exit 1
elif [ -f $cert -a -f $key ]
then
printf "${NC}${name} cert and key found, verifying the authenticity\n"
CERT_SUBJECT=$(sudo openssl x509 -in $cert -text | grep "Subject: CN"| tr -d " ")
CERT_ISSUER=$(sudo openssl x509 -in $cert -text | grep "Issuer: CN"| tr -d " ")
CERT_MD5=$(sudo openssl x509 -noout -modulus -in $cert | openssl md5| awk '{print $2}')
KEY_MD5=$(sudo openssl rsa -noout -modulus -in $key | openssl md5| awk '{print $2}')
if [ $CERT_SUBJECT == "${subject}" ] && [ $CERT_ISSUER == "${issuer}" ] && [ $CERT_MD5 == $KEY_MD5 ]
then
printf "${SUCCESS}${name} cert and key are correct\n${NC}"
else
printf "${FAILED}Exiting...Found mismtach in the ${name} certificate and keys, More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/04-certificate-authority.md#certificate-authority\n${NC}"
exit 1
fi
else
printf "${FAILED}${cert} / ${key} is missing. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/04-certificate-authority.md#certificate-authority\n"
echo "These should be in /var/lib/kubernetes/pki (most certs), /etc/etcd (eccd server certs) or /var/lib/kubelet (kubelet certs)${NC}"
exit 1
fi
}
check_cert_only()
{
local name=$1
local subject=$2
local issuer=$3
local cert="${CERT_LOCATION}/$1.crt"
# Worker-2 auto cert is a .pem
[ -f "${CERT_LOCATION}/$1.pem" ] && cert="${CERT_LOCATION}/$1.pem"
if [ -z $cert ]
then
printf "${FAILED}cert not present in ${CERT_LOCATION}. Perhaps you missed a copy step\n${NC}"
exit 1
elif [ -f $cert ]
then
printf "${NC}${name} cert found, verifying the authenticity\n"
CERT_SUBJECT=$(sudo openssl x509 -in $cert -text | grep "Subject: "| tr -d " ")
CERT_ISSUER=$(sudo openssl x509 -in $cert -text | grep "Issuer: CN"| tr -d " ")
CERT_MD5=$(sudo openssl x509 -noout -modulus -in $cert | openssl md5| awk '{print $2}')
if [ $CERT_SUBJECT == "${subject}" ] && [ $CERT_ISSUER == "${issuer}" ]
then
printf "${SUCCESS}${name} cert is correct\n${NC}"
else
printf "${FAILED}Exiting...Found mismtach in the ${name} certificate, More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/04-certificate-authority.md#certificate-authority\n${NC}"
exit 1
fi
else
if [[ $cert == *kubelet-client-current* ]]
then
printf "${FAILED}${cert} missing. This probably means that kubelet failed to start.${NC}\n"
echo -e "Check logs with\n\n sudo journalctl -u kubelet\n"
else
printf "${FAILED}${cert} missing. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/04-certificate-authority.md#certificate-authority\n${NC}"
echo "These should be in ${CERT_LOCATION}"
fi
exit 1
fi
}
check_cert_adminkubeconfig()
{
if [ -z $ADMINKUBECONFIG ]
then
printf "${FAILED}please specify admin kubeconfig location\n${NC}"
exit 1
elif [ -f $ADMINKUBECONFIG ]
then
printf "${NC}admin kubeconfig file found, verifying the authenticity\n"
ADMINKUBECONFIG_SUBJECT=$(cat $ADMINKUBECONFIG | grep "client-certificate-data:" | awk '{print $2}' | base64 --decode | sudo openssl x509 -text | grep "Subject: CN" | tr -d " ")
ADMINKUBECONFIG_ISSUER=$(cat $ADMINKUBECONFIG | grep "client-certificate-data:" | awk '{print $2}' | base64 --decode | sudo openssl x509 -text | grep "Issuer: CN" | tr -d " ")
ADMINKUBECONFIG_CERT_MD5=$(cat $ADMINKUBECONFIG | grep "client-certificate-data:" | awk '{print $2}' | base64 --decode | sudo openssl x509 -noout | openssl md5 | awk '{print $2}')
ADMINKUBECONFIG_KEY_MD5=$(cat $ADMINKUBECONFIG | grep "client-key-data" | awk '{print $2}' | base64 --decode | openssl rsa -noout | openssl md5 | awk '{print $2}')
ADMINKUBECONFIG_SERVER=$(cat $ADMINKUBECONFIG | grep "server:"| awk '{print $2}')
if [ $ADMINKUBECONFIG_SUBJECT == "Subject:CN=admin,O=system:masters" ] && [ $ADMINKUBECONFIG_ISSUER == "Issuer:CN=KUBERNETES-CA,O=Kubernetes" ] && [ $ADMINKUBECONFIG_CERT_MD5 == $ADMINKUBECONFIG_KEY_MD5 ] && [ $ADMINKUBECONFIG_SERVER == "https://127.0.0.1:6443" ]
then
printf "${SUCCESS}admin kubeconfig cert and key are correct\n"
else
printf "${FAILED}Exiting...Found mismtach in the admin kubeconfig certificate and keys, More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/05-kubernetes-configuration-files.md#the-admin-kubernetes-configuration-file\n"
exit 1
fi
else
printf "${FAILED}admin kubeconfig file is missing. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/05-kubernetes-configuration-files.md#the-admin-kubernetes-configuration-file\n"
exit 1
fi
}
get_kubeconfig_cert_path()
{
local kubeconfig=$1
local cert_field=$2
sudo cat $kubeconfig | grep cert_field | awk '{print $2}'
}
check_kubeconfig()
{
local name=$1
local location=$2
local apiserver=$3
local kubeconfig="${location}/${name}.kubeconfig"
echo "Checking $kubeconfig"
check_kubeconfig_exists $name $location
ca=$(get_kubeconfig_cert_path $kubeconfig "certificate-authority")
cert=$(get_kubeconfig_cert_path $kubeconfig "client-certificate")
key=$(get_kubeconfig_cert_path $kubeconfig "client-key")
server=$(sudo cat $kubeconfig | grep server | awk '{print $2}')
if [ -f "$ca"]
then
printf "${SUCCESS}Path to CA certificate is correct${NC}\n"
else
printf "${FAIL}CA certificate not found at ${ca}${NC}\n"
exit 1
fi
if [ -f "$cert"]
then
printf "${SUCCESS}Path to client certificate is correct${NC}\n"
else
printf "${FAIL}Client certificate not found at ${cert}${NC}\n"
exit 1
fi
if [ -f "$key"]
then
printf "${SUCCESS}Path to client key is correct${NC}\n"
else
printf "${FAIL}Client key not found at ${key}${NC}\n"
exit 1
fi
if [ "$apiserver" = "$server" ]
then
printf "${SUCCESS}Server URL is correct${NC}\n"
else
printf "${FAIL}Server URL ${server} is incorrect${NC}\n"
exit 1
fi
}
check_kubeconfig_exists() {
local name=$1
local location=$2
local kubeconfig="${location}/${name}.kubeconfig"
if [ -f "${kubeconfig}" ]
then
printf "${SUCCESS}${kubeconfig} found${NC}\n"
else
printf "${FAIL}${kubeconfig} not found!${NC}\n"
exit 1
fi
}
check_systemd_etcd()
{
if [ -z $ETCDCERT ] && [ -z $ETCDKEY ]
then
printf "${FAILED}please specify ETCD cert and key location, Exiting....\n${NC}"
exit 1
elif [ -f $SYSTEMD_ETCD_FILE ]
then
printf "${NC}Systemd for ETCD service found, verifying the authenticity\n"
# Systemd cert and key file details
ETCD_CA_CERT=ca.crt
CERT_FILE=$(systemctl cat etcd.service | grep "\--cert-file"| awk '{print $1}'| cut -d "=" -f2)
KEY_FILE=$(systemctl cat etcd.service | grep "\--key-file"| awk '{print $1}' | cut -d "=" -f2)
PEER_CERT_FILE=$(systemctl cat etcd.service | grep "\--peer-cert-file"| awk '{print $1}'| cut -d "=" -f2)
PEER_KEY_FILE=$(systemctl cat etcd.service | grep "\--peer-key-file"| awk '{print $1}'| cut -d "=" -f2)
TRUSTED_CA_FILE=$(systemctl cat etcd.service | grep "\--trusted-ca-file"| awk '{print $1}'| cut -d "=" -f2)
PEER_TRUSTED_CA_FILE=$(systemctl cat etcd.service | grep "\--peer-trusted-ca-file"| awk '{print $1}'| cut -d "=" -f2)
# Systemd advertise , client and peer url's
IAP_URL=$(systemctl cat etcd.service | grep "\--initial-advertise-peer-urls"| awk '{print $2}')
LP_URL=$(systemctl cat etcd.service | grep "\--listen-peer-urls"| awk '{print $2}')
LC_URL=$(systemctl cat etcd.service | grep "\--listen-client-urls"| awk '{print $2}')
AC_URL=$(systemctl cat etcd.service | grep "\--advertise-client-urls"| awk '{print $2}')
ETCD_CA_CERT=/etc/etcd/ca.crt
ETCDCERT=/etc/etcd/etcd-server.crt
ETCDKEY=/etc/etcd/etcd-server.key
if [ $CERT_FILE == $ETCDCERT ] && [ $KEY_FILE == $ETCDKEY ] && [ $PEER_CERT_FILE == $ETCDCERT ] && [ $PEER_KEY_FILE == $ETCDKEY ] && \
[ $TRUSTED_CA_FILE == $ETCD_CA_CERT ] && [ $PEER_TRUSTED_CA_FILE = $ETCD_CA_CERT ]
then
printf "${SUCCESS}ETCD certificate, ca and key files are correct under systemd service\n${NC}"
else
printf "${FAILED}Exiting...Found mismtach in the ETCD certificate, ca and keys. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/07-bootstrapping-etcd.md#configure-the-etcd-server\n${NC}"
exit 1
fi
if [ $IAP_URL == "https://$PRIMARY_IP:2380" ] && [ $LP_URL == "https://$PRIMARY_IP:2380" ] && [ $LC_URL == "https://$PRIMARY_IP:2379,https://127.0.0.1:2379" ] && \
[ $AC_URL == "https://$PRIMARY_IP:2379" ]
then
printf "${SUCCESS}ETCD initial-advertise-peer-urls, listen-peer-urls, listen-client-urls, advertise-client-urls are correct\n${NC}"
else
printf "${FAILED}Exiting...Found mismtach in the ETCD initial-advertise-peer-urls / listen-peer-urls / listen-client-urls / advertise-client-urls. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/07-bootstrapping-etcd.md#configure-the-etcd-server\n${NC}"
exit 1
fi
else
printf "${FAILED}etcd-server.crt / etcd-server.key is missing. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/07-bootstrapping-etcd.md#configure-the-etcd-server\n${NC}"
exit 1
fi
}
check_systemd_api()
{
if [ -z $APICERT ] && [ -z $APIKEY ]
then
printf "${FAILED}please specify kube-api cert and key location, Exiting....\n${NC}"
exit 1
elif [ -f $SYSTEMD_API_FILE ]
then
printf "Systemd for kube-api service found, verifying the authenticity\n"
ADVERTISE_ADDRESS=$(systemctl cat kube-apiserver.service | grep "\--advertise-address" | awk '{print $1}' | cut -d "=" -f2)
CLIENT_CA_FILE=$(systemctl cat kube-apiserver.service | grep "\--client-ca-file" | awk '{print $1}' | cut -d "=" -f2)
ETCD_CA_FILE=$(systemctl cat kube-apiserver.service | grep "\--etcd-cafile" | awk '{print $1}' | cut -d "=" -f2)
ETCD_CERT_FILE=$(systemctl cat kube-apiserver.service | grep "\--etcd-certfile" | awk '{print $1}' | cut -d "=" -f2)
ETCD_KEY_FILE=$(systemctl cat kube-apiserver.service | grep "\--etcd-keyfile" | awk '{print $1}' | cut -d "=" -f2)
KUBELET_CERTIFICATE_AUTHORITY=$(systemctl cat kube-apiserver.service | grep "\--kubelet-certificate-authority" | awk '{print $1}' | cut -d "=" -f2)
KUBELET_CLIENT_CERTIFICATE=$(systemctl cat kube-apiserver.service | grep "\--kubelet-client-certificate" | awk '{print $1}' | cut -d "=" -f2)
KUBELET_CLIENT_KEY=$(systemctl cat kube-apiserver.service | grep "\--kubelet-client-key" | awk '{print $1}' | cut -d "=" -f2)
SERVICE_ACCOUNT_KEY_FILE=$(systemctl cat kube-apiserver.service | grep "\--service-account-key-file" | awk '{print $1}' | cut -d "=" -f2)
TLS_CERT_FILE=$(systemctl cat kube-apiserver.service | grep "\--tls-cert-file" | awk '{print $1}' | cut -d "=" -f2)
TLS_PRIVATE_KEY_FILE=$(systemctl cat kube-apiserver.service | grep "\--tls-private-key-file" | awk '{print $1}' | cut -d "=" -f2)
PKI=/var/lib/kubernetes/pki
CACERT="${PKI}/ca.crt"
APICERT="${PKI}/kube-apiserver.crt"
APIKEY="${PKI}/kube-apiserver.key"
SACERT="${PKI}/service-account.crt"
KCCERT="${PKI}/apiserver-kubelet-client.crt"
KCKEY="${PKI}/apiserver-kubelet-client.key"
if [ $ADVERTISE_ADDRESS == $PRIMARY_IP ] && [ $CLIENT_CA_FILE == $CACERT ] && [ $ETCD_CA_FILE == $CACERT ] && \
[ $ETCD_CERT_FILE == "${PKI}/etcd-server.crt" ] && [ $ETCD_KEY_FILE == "${PKI}/etcd-server.key" ] && \
[ $KUBELET_CERTIFICATE_AUTHORITY == $CACERT ] && [ $KUBELET_CLIENT_CERTIFICATE == $KCCERT ] && [ $KUBELET_CLIENT_KEY == $KCKEY ] && \
[ $SERVICE_ACCOUNT_KEY_FILE == $SACERT ] && [ $TLS_CERT_FILE == $APICERT ] && [ $TLS_PRIVATE_KEY_FILE == $APIKEY ]
then
printf "${SUCCESS}kube-apiserver advertise-address/ client-ca-file/ etcd-cafile/ etcd-certfile/ etcd-keyfile/ kubelet-certificate-authority/ kubelet-client-certificate/ kubelet-client-key/ service-account-key-file/ tls-cert-file/ tls-private-key-file are correct\n${NC}"
else
printf "${FAILED}Exiting...Found mismtach in the kube-apiserver systemd file, check advertise-address/ client-ca-file/ etcd-cafile/ etcd-certfile/ etcd-keyfile/ kubelet-certificate-authority/ kubelet-client-certificate/ kubelet-client-key/ service-account-key-file/ tls-cert-file/ tls-private-key-file. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/08-bootstrapping-kubernetes-controllers.md#configure-the-kubernetes-api-server\n${NC}"
exit 1
fi
else
printf "${FAILED}kube-apiserver.crt / kube-apiserver.key is missing. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/08-bootstrapping-kubernetes-controllers.md#configure-the-kubernetes-api-server\n${NC}"
exit 1
fi
}
check_systemd_kcm()
{
KCMCERT=/var/lib/kubernetes/pki/kube-controller-manager.crt
KCMKEY=/var/lib/kubernetes/pki/kube-controller-manager.key
CACERT=/var/lib/kubernetes/pki/ca.crt
CAKEY=/var/lib/kubernetes/pki/ca.key
SAKEY=/var/lib/kubernetes/pki/service-account.key
KCMKUBECONFIG=/var/lib/kubernetes/kube-controller-manager.kubeconfig
if [ -z $KCMCERT ] && [ -z $KCMKEY ]
then
printf "${FAILED}please specify cert and key location\n${NC}"
exit 1
elif [ -f $SYSTEMD_KCM_FILE ]
then
printf "Systemd for kube-controller-manager service found, verifying the authenticity\n"
CLUSTER_SIGNING_CERT_FILE=$(systemctl cat kube-controller-manager.service | grep "\--cluster-signing-cert-file" | awk '{print $1}' | cut -d "=" -f2)
CLUSTER_SIGNING_KEY_FILE=$(systemctl cat kube-controller-manager.service | grep "\--cluster-signing-key-file" | awk '{print $1}' | cut -d "=" -f2)
KUBECONFIG=$(systemctl cat kube-controller-manager.service | grep "\--kubeconfig" | awk '{print $1}' | cut -d "=" -f2)
ROOT_CA_FILE=$(systemctl cat kube-controller-manager.service | grep "\--root-ca-file" | awk '{print $1}' | cut -d "=" -f2)
SERVICE_ACCOUNT_PRIVATE_KEY_FILE=$(systemctl cat kube-controller-manager.service | grep "\--service-account-private-key-file" | awk '{print $1}' | cut -d "=" -f2)
if [ $CLUSTER_SIGNING_CERT_FILE == $CACERT ] && [ $CLUSTER_SIGNING_KEY_FILE == $CAKEY ] && [ $KUBECONFIG == $KCMKUBECONFIG ] && \
[ $ROOT_CA_FILE == $CACERT ] && [ $SERVICE_ACCOUNT_PRIVATE_KEY_FILE == $SAKEY ]
then
printf "${SUCCESS}kube-controller-manager cluster-signing-cert-file, cluster-signing-key-file, kubeconfig, root-ca-file, service-account-private-key-file are correct\n${NC}"
else
printf "${FAILED}Exiting...Found mismtach in the kube-controller-manager cluster-signing-cert-file, cluster-signing-key-file, kubeconfig, root-ca-file, service-account-private-key-file. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/08-bootstrapping-kubernetes-controllers.md#configure-the-kubernetes-controller-manager\n${NC}"
exit 1
fi
else
printf "${FAILED}kube-controller-manager.crt / kube-controller-manager.key is missing. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/08-bootstrapping-kubernetes-controllers.md#configure-the-kubernetes-controller-manager\n${NC}"
exit 1
fi
}
check_systemd_ks()
{
KSCERT=/var/lib/kubernetes/pki/kube-scheduler.crt
KSKEY=/var/lib/kubernetes/pki/kube-scheduler.key
KSKUBECONFIG=/var/lib/kubernetes/kube-scheduler.kubeconfig
if [ -z $KSCERT ] && [ -z $KSKEY ]
then
printf "${FAILED}please specify cert and key location\n${NC}"
exit 1
elif [ -f $SYSTEMD_KS_FILE ]
then
printf "Systemd for kube-scheduler service found, verifying the authenticity\n"
KUBECONFIG=$(systemctl cat kube-scheduler.service | grep "\--kubeconfig"| awk '{print $1}'| cut -d "=" -f2)
if [ $KUBECONFIG == $KSKUBECONFIG ]
then
printf "${SUCCESS}kube-scheduler --kubeconfig is correct\n${NC}"
else
printf "${FAILED}Exiting...Found mismtach in the kube-scheduler --kubeconfig. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/08-bootstrapping-kubernetes-controllers.md#configure-the-kubernetes-scheduler\n${NC}"
exit 1
fi
else
printf "${FAILED}kube-scheduler.crt / kube-scheduler.key is missing. More details: https://github.com/mmumshad/kubernetes-the-hard-way/blob/master/docs/08-bootstrapping-kubernetes-controllers.md#configure-the-kubernetes-scheduler\n${NC}"
exit 1
fi
}
# END OF Function - Master node #
if [ ! -z "$1" ]
then
choice=$1
else
echo "This script will validate the certificates in master as well as node01 nodes. Before proceeding, make sure you ssh into the respective node [ Master or Worker-1 ] for certificate validation"
while true
do
echo
echo " 1. Verify certificates on Master Nodes after step 4"
echo " 2. Verify kubeconfigs on Master Nodes after step 5"
echo " 3. Verify kubeconfigs and PKI on Master Nodes after step 8"
echo " 4. Verify kubeconfigs and PKI on node01 Node after step 10"
echo " 5. Verify kubeconfigs and PKI on node02 Node after step 11"
echo
echo -n "Please select one of the above options: "
read choice
[ -z "$choice" ] && continue
[ $choice -gt 0 -a $choice -lt 6 ] && break
done
fi
HOST=$(hostname -s)
CERT_ISSUER="Issuer:CN=KUBERNETES-CA,O=Kubernetes"
SUBJ_CA="Subject:CN=KUBERNETES-CA,O=Kubernetes"
SUBJ_ADMIN="Subject:CN=admin,O=system:masters"
SUBJ_KCM="Subject:CN=system:kube-controller-manager,O=system:kube-controller-manager"
SUBJ_KP="Subject:CN=system:kube-proxy,O=system:node-proxier"
SUBJ_KS="Subject:CN=system:kube-scheduler,O=system:kube-scheduler"
SUBJ_API="Subject:CN=kube-apiserver,O=Kubernetes"
SUBJ_SA="Subject:CN=service-accounts,O=Kubernetes"
SUBJ_ETCD="Subject:CN=etcd-server,O=Kubernetes"
SUBJ_APIKC="Subject:CN=kube-apiserver-kubelet-client,O=system:masters"
case $choice in
1)
if ! [ "${HOST}" = "controlplane01" -o "${HOST}" = "controlplane02" ]
then
printf "${FAILED}Must run on controlplane01 or controlplane02${NC}\n"
exit 1
fi
echo -e "The selected option is $choice, proceeding the certificate verification of Master node"
CERT_LOCATION=$HOME
check_cert_and_key "ca" $SUBJ_CA $CERT_ISSUER
check_cert_and_key "kube-apiserver" $SUBJ_API $CERT_ISSUER
check_cert_and_key "kube-controller-manager" $SUBJ_KCM $CERT_ISSUER
check_cert_and_key "kube-scheduler" $SUBJ_KS $CERT_ISSUER
check_cert_and_key "service-account" $SUBJ_SA $CERT_ISSUER
check_cert_and_key "apiserver-kubelet-client" $SUBJ_APIKC $CERT_ISSUER
check_cert_and_key "etcd-server" $SUBJ_ETCD $CERT_ISSUER
if [ "${HOST}" = "controlplane01" ]
then
check_cert_and_key "admin" $SUBJ_ADMIN $CERT_ISSUER
check_cert_and_key "kube-proxy" $SUBJ_KP $CERT_ISSUER
fi
;;
2)
if ! [ "${HOST}" = "controlplane01" -o "${HOST}" = "controlplane02" ]
then
printf "${FAILED}Must run on controlplane01 or controlplane02${NC}\n"
exit 1
fi
check_cert_adminkubeconfig
check_kubeconfig_exists "kube-controller-manager" $HOME
check_kubeconfig_exists "kube-scheduler" $HOME
if [ "${HOST}" = "controlplane01" ]
then
check_kubeconfig_exists "kube-proxy" $HOME
fi
;;
3)
if ! [ "${HOST}" = "controlplane01" -o "${HOST}" = "controlplane02" ]
then
printf "${FAILED}Must run on controlplane01 or controlplane02${NC}\n"
exit 1
fi
CERT_LOCATION=/etc/etcd
check_cert_only "ca" $SUBJ_CA $CERT_ISSUER
check_cert_and_key "etcd-server" $SUBJ_ETCD $CERT_ISSUER
CERT_LOCATION=/var/lib/kubernetes/pki
check_cert_and_key "ca" $SUBJ_CA $CERT_ISSUER
check_cert_and_key "kube-apiserver" $SUBJ_API $CERT_ISSUER
check_cert_and_key "kube-controller-manager" $SUBJ_KCM $CERT_ISSUER
check_cert_and_key "kube-scheduler" $SUBJ_KS $CERT_ISSUER
check_cert_and_key "service-account" $SUBJ_SA $CERT_ISSUER
check_cert_and_key "apiserver-kubelet-client" $SUBJ_APIKC $CERT_ISSUER
check_cert_and_key "etcd-server" $SUBJ_ETCD $CERT_ISSUER
check_kubeconfig "kube-controller-manager" "/var/lib/kubernetes" "https://127.0.0.1:6443"
check_kubeconfig "kube-scheduler" "/var/lib/kubernetes" "https://127.0.0.1:6443"
check_systemd_api
check_systemd_etcd
check_systemd_kcm
check_systemd_ks
;;
4)
if ! [ "${HOST}" = "node01" ]
then
printf "${FAILED}Must run on node01${NC}\n"
exit 1
fi
CERT_LOCATION=/var/lib/kubernetes/pki
check_cert_only "ca" $SUBJ_CA $CERT_ISSUER
check_cert_and_key "kube-proxy" $SUBJ_KP $CERT_ISSUER
check_cert_and_key "node01" "Subject:CN=system:node:node01,O=system:nodes" $CERT_ISSUER
check_kubeconfig "kube-proxy" "/var/lib/kube-proxy" "https://${LOADBALANCER}:6443"
check_kubeconfig "kubelet" "/var/lib/kubelet" "https://${LOADBALANCER}:6443"
;;
5)
if ! [ "${HOST}" = "node02" ]
then
printf "${FAILED}Must run on node02${NC}\n"
exit 1
fi
CERT_LOCATION=/var/lib/kubernetes/pki
check_cert_only "ca" $SUBJ_CA $CERT_ISSUER
check_cert_and_key "kube-proxy" $SUBJ_KP $CERT_ISSUER
CERT_LOCATION=/var/lib/kubelet/pki
check_cert_only "kubelet-client-current" "Subject:O=system:nodes,CN=system:node:node02" $CERT_ISSUER
check_kubeconfig "kube-proxy" "/var/lib/kube-proxy" "https://${LOADBALANCER}:6443"
;;
*)
printf "${FAILED}Exiting.... Please select the valid option either 1 or 2\n${NC}"
exit 1
;;
esac