From 2adb5c0f5cae7e9d3129a4d8ab9f2ff8daf8ffaf Mon Sep 17 00:00:00 2001 From: Maksymilian Mulawa Date: Thu, 2 Feb 2023 14:04:04 +0100 Subject: [PATCH] Update to Kubernetes 1.26.1 on Ubuntu 22.04 --- README.md | 11 +- deployments/coredns-1.9.3.yaml | 187 ++++++++++++++++++ docs/01-prerequisites.md | 2 +- docs/02-client-tools.md | 8 +- docs/03-compute-resources.md | 8 +- docs/07-bootstrapping-etcd.md | 6 +- ...08-bootstrapping-kubernetes-controllers.md | 32 +-- docs/09-bootstrapping-kubernetes-workers.md | 49 +++-- docs/10-configuring-kubectl.md | 10 +- docs/12-dns-addon.md | 2 +- docs/13-smoke-test.md | 22 +-- 11 files changed, 265 insertions(+), 72 deletions(-) create mode 100644 deployments/coredns-1.9.3.yaml diff --git a/README.md b/README.md index 4043b02..42e5734 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,12 @@ The target audience for this tutorial is someone planning to support a productio Kubernetes The Hard Way guides you through bootstrapping a highly available Kubernetes cluster with end-to-end encryption between components and RBAC authentication. -* [kubernetes](https://github.com/kubernetes/kubernetes) v1.21.0 -* [containerd](https://github.com/containerd/containerd) v1.4.4 -* [coredns](https://github.com/coredns/coredns) v1.8.3 -* [cni](https://github.com/containernetworking/cni) v0.9.1 -* [etcd](https://github.com/etcd-io/etcd) v3.4.15 +* [kubernetes](https://github.com/kubernetes/kubernetes) v1.26.1 +* [containerd](https://github.com/containerd/containerd) v1.6.15 +* [runc](https://github.com/opencontainers/runc) v1.1.4 +* [coredns](https://github.com/coredns/coredns) v1.9.3 +* [cni](https://github.com/containernetworking/cni) v1.2.0 +* [etcd](https://github.com/etcd-io/etcd) v3.5.6 ## Labs diff --git a/deployments/coredns-1.9.3.yaml b/deployments/coredns-1.9.3.yaml new file mode 100644 index 0000000..e5667ec --- /dev/null +++ b/deployments/coredns-1.9.3.yaml @@ -0,0 +1,187 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + Corefile: | + .:53 { + errors + health + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + cache 30 + loop + reload + loadbalance + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + nodeSelector: + kubernetes.io/os: linux + containers: + - name: coredns + image: coredns/coredns:1.9.3 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.32.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP diff --git a/docs/01-prerequisites.md b/docs/01-prerequisites.md index d2474ce..abb1067 100644 --- a/docs/01-prerequisites.md +++ b/docs/01-prerequisites.md @@ -14,7 +14,7 @@ This tutorial leverages the [Google Cloud Platform](https://cloud.google.com/) t Follow the Google Cloud SDK [documentation](https://cloud.google.com/sdk/) to install and configure the `gcloud` command line utility. -Verify the Google Cloud SDK version is 338.0.0 or higher: +Verify the Google Cloud SDK version is 416.0.0 or higher: ``` gcloud version diff --git a/docs/02-client-tools.md b/docs/02-client-tools.md index 94d93be..091d14e 100644 --- a/docs/02-client-tools.md +++ b/docs/02-client-tools.md @@ -76,7 +76,7 @@ The `kubectl` command line utility is used to interact with the Kubernetes API S ### OS X ``` -curl -o kubectl https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/darwin/amd64/kubectl +curl -o kubectl https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/darwin/amd64/kubectl ``` ``` @@ -90,7 +90,7 @@ sudo mv kubectl /usr/local/bin/ ### Linux ``` -wget https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl +wget https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl ``` ``` @@ -103,7 +103,7 @@ sudo mv kubectl /usr/local/bin/ ### Verification -Verify `kubectl` version 1.21.0 or higher is installed: +Verify `kubectl` version 1.26.1 or higher is installed: ``` kubectl version --client @@ -112,7 +112,7 @@ kubectl version --client > output ``` -Client Version: version.Info{Major:"1", Minor:"21", GitVersion:"v1.21.0", GitCommit:"cb303e613a121a29364f75cc67d3d580833a7479", GitTreeState:"clean", BuildDate:"2021-04-08T16:31:21Z", GoVersion:"go1.16.1", Compiler:"gc", Platform:"linux/amd64"} +Client Version: version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.1", GitCommit:"8f94681cd294aa8cfd3407b8191f6c70214973a4", GitTreeState:"clean", BuildDate:"2023-01-18T15:58:16Z", GoVersion:"go1.19.5", Compiler:"gc", Platform:"linux/amd64"} ``` Next: [Provisioning Compute Resources](03-compute-resources.md) diff --git a/docs/03-compute-resources.md b/docs/03-compute-resources.md index a5402bb..ce34a6d 100644 --- a/docs/03-compute-resources.md +++ b/docs/03-compute-resources.md @@ -92,7 +92,7 @@ kubernetes-the-hard-way XX.XXX.XXX.XXX EXTERNAL us-west1 ## Compute Instances -The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 20.04, which has good support for the [containerd container runtime](https://github.com/containerd/containerd). Each compute instance will be provisioned with a fixed private IP address to simplify the Kubernetes bootstrapping process. +The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 22.04, which has good support for the [containerd container runtime](https://github.com/containerd/containerd). Each compute instance will be provisioned with a fixed private IP address to simplify the Kubernetes bootstrapping process. ### Kubernetes Controllers @@ -104,7 +104,7 @@ for i in 0 1 2; do --async \ --boot-disk-size 200GB \ --can-ip-forward \ - --image-family ubuntu-2004-lts \ + --image-family ubuntu-2204-lts \ --image-project ubuntu-os-cloud \ --machine-type e2-standard-2 \ --private-network-ip 10.240.0.1${i} \ @@ -128,7 +128,7 @@ for i in 0 1 2; do --async \ --boot-disk-size 200GB \ --can-ip-forward \ - --image-family ubuntu-2004-lts \ + --image-family ubuntu-2204-lts \ --image-project ubuntu-os-cloud \ --machine-type e2-standard-2 \ --metadata pod-cidr=10.200.${i}.0/24 \ @@ -208,7 +208,7 @@ Waiting for SSH key to propagate. After the SSH keys have been updated you'll be logged into the `controller-0` instance: ``` -Welcome to Ubuntu 20.04.2 LTS (GNU/Linux 5.4.0-1042-gcp x86_64) +Welcome to Ubuntu 22.04.1 LTS (GNU/Linux 5.15.0-1027-gcp x86_64) ... ``` diff --git a/docs/07-bootstrapping-etcd.md b/docs/07-bootstrapping-etcd.md index a9ad937..a29a4b1 100644 --- a/docs/07-bootstrapping-etcd.md +++ b/docs/07-bootstrapping-etcd.md @@ -22,15 +22,15 @@ Download the official etcd release binaries from the [etcd](https://github.com/e ``` wget -q --show-progress --https-only --timestamping \ - "https://github.com/etcd-io/etcd/releases/download/v3.4.15/etcd-v3.4.15-linux-amd64.tar.gz" + "https://github.com/etcd-io/etcd/releases/download/v3.5.6/etcd-v3.5.6-linux-amd64.tar.gz" ``` Extract and install the `etcd` server and the `etcdctl` command line utility: ``` { - tar -xvf etcd-v3.4.15-linux-amd64.tar.gz - sudo mv etcd-v3.4.15-linux-amd64/etcd* /usr/local/bin/ + tar -xvf etcd-v3.5.6-linux-amd64.tar.gz + sudo mv etcd-v3.5.6-linux-amd64/etcd* /usr/local/bin/ } ``` diff --git a/docs/08-bootstrapping-kubernetes-controllers.md b/docs/08-bootstrapping-kubernetes-controllers.md index ede6ce0..aef06e5 100644 --- a/docs/08-bootstrapping-kubernetes-controllers.md +++ b/docs/08-bootstrapping-kubernetes-controllers.md @@ -28,10 +28,10 @@ Download the official Kubernetes release binaries: ``` wget -q --show-progress --https-only --timestamping \ - "https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kube-apiserver" \ - "https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kube-controller-manager" \ - "https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kube-scheduler" \ - "https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl" + "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kube-apiserver" \ + "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kube-controller-manager" \ + "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kube-scheduler" \ + "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl" ``` Install the Kubernetes binaries: @@ -170,7 +170,7 @@ Create the `kube-scheduler.yaml` configuration file: ``` cat < Allow up to 10 seconds for the Kubernetes API Server to fully initialize. +Check if all Controller Services are active and are running +``` +sudo systemctl status kube-apiserver kube-controller-manager kube-scheduler +``` + ### Enable HTTP Health Checks A [Google Network Load Balancer](https://cloud.google.com/compute/docs/load-balancing/network) will be used to distribute traffic across the three API servers and allow each API server to terminate TLS connections and validate client certificates. The network load balancer only supports HTTP health checks which means the HTTPS endpoint exposed by the API server cannot be used. As a workaround the nginx webserver can be used to proxy HTTP health checks. In this section nginx will be installed and configured to accept HTTP health checks on port `80` and proxy the connections to the API server on `https://127.0.0.1:6443/healthz`. @@ -274,14 +279,15 @@ curl -H "Host: kubernetes.default.svc.cluster.local" -i http://127.0.0.1/healthz ``` HTTP/1.1 200 OK Server: nginx/1.18.0 (Ubuntu) -Date: Sun, 02 May 2021 04:19:29 GMT +Date: Thu, 02 Feb 2023 12:08:51 GMT Content-Type: text/plain; charset=utf-8 Content-Length: 2 Connection: keep-alive +Audit-Id: 189350ca-dfed-4b14-83f6-36e97f598f44 Cache-Control: no-cache, private X-Content-Type-Options: nosniff -X-Kubernetes-Pf-Flowschema-Uid: c43f32eb-e038-457f-9474-571d43e5c325 -X-Kubernetes-Pf-Prioritylevel-Uid: 8ba5908f-5569-4330-80fd-c643e7512366 +X-Kubernetes-Pf-Flowschema-Uid: 93e62fd3-1ca0-4262-9975-8a40fc9f199d +X-Kubernetes-Pf-Prioritylevel-Uid: 695ba920-5ea5-45c3-8406-1fa570d9d8af ok ``` @@ -412,12 +418,12 @@ curl --cacert ca.pem https://${KUBERNETES_PUBLIC_ADDRESS}:6443/version ``` { "major": "1", - "minor": "21", - "gitVersion": "v1.21.0", - "gitCommit": "cb303e613a121a29364f75cc67d3d580833a7479", + "minor": "26", + "gitVersion": "v1.26.1", + "gitCommit": "8f94681cd294aa8cfd3407b8191f6c70214973a4", "gitTreeState": "clean", - "buildDate": "2021-04-08T16:25:06Z", - "goVersion": "go1.16.1", + "buildDate": "2023-01-18T15:51:25Z", + "goVersion": "go1.19.5", "compiler": "gc", "platform": "linux/amd64" } diff --git a/docs/09-bootstrapping-kubernetes-workers.md b/docs/09-bootstrapping-kubernetes-workers.md index 9958f88..8dfb7cc 100644 --- a/docs/09-bootstrapping-kubernetes-workers.md +++ b/docs/09-bootstrapping-kubernetes-workers.md @@ -49,13 +49,13 @@ sudo swapoff -a ``` wget -q --show-progress --https-only --timestamping \ - https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.21.0/crictl-v1.21.0-linux-amd64.tar.gz \ - https://github.com/opencontainers/runc/releases/download/v1.0.0-rc93/runc.amd64 \ - https://github.com/containernetworking/plugins/releases/download/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz \ - https://github.com/containerd/containerd/releases/download/v1.4.4/containerd-1.4.4-linux-amd64.tar.gz \ - https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl \ - https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kube-proxy \ - https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-linux-amd64.tar.gz \ + https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64 \ + https://github.com/containernetworking/plugins/releases/download/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz \ + https://github.com/containerd/containerd/releases/download/v1.6.15/containerd-1.6.15-linux-amd64.tar.gz \ + https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl \ + https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kube-proxy \ + https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubelet ``` Create the installation directories: @@ -75,9 +75,9 @@ Install the worker binaries: ``` { mkdir containerd - tar -xvf crictl-v1.21.0-linux-amd64.tar.gz - tar -xvf containerd-1.4.4-linux-amd64.tar.gz -C containerd - sudo tar -xvf cni-plugins-linux-amd64-v0.9.1.tgz -C /opt/cni/bin/ + tar -xvf crictl-v1.26.0-linux-amd64.tar.gz + tar -xvf containerd-1.6.15-linux-amd64.tar.gz -C containerd + sudo tar -xvf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/ sudo mv runc.amd64 runc chmod +x crictl kubectl kube-proxy kubelet runc sudo mv crictl kubectl kube-proxy kubelet runc /usr/local/bin/ @@ -137,15 +137,10 @@ sudo mkdir -p /etc/containerd/ ``` ``` -cat << EOF | sudo tee /etc/containerd/config.toml -[plugins] - [plugins.cri.containerd] - snapshotter = "overlayfs" - [plugins.cri.containerd.default_runtime] - runtime_type = "io.containerd.runtime.v1.linux" - runtime_engine = "/usr/local/bin/runc" - runtime_root = "" -EOF +{ + containerd config default | sudo tee /etc/containerd/config.toml + sudo sed -i 's/SystemdCgroup \= false/SystemdCgroup \= true/g' /etc/containerd/config.toml +} ``` Create the `containerd.service` systemd unit file: @@ -199,6 +194,7 @@ authentication: clientCAFile: "/var/lib/kubernetes/ca.pem" authorization: mode: Webhook +cgroupDriver: systemd clusterDomain: "cluster.local" clusterDNS: - "10.32.0.10" @@ -225,11 +221,8 @@ Requires=containerd.service [Service] ExecStart=/usr/local/bin/kubelet \\ --config=/var/lib/kubelet/kubelet-config.yaml \\ - --container-runtime=remote \\ --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock \\ - --image-pull-progress-deadline=2m \\ --kubeconfig=/var/lib/kubelet/kubeconfig \\ - --network-plugin=cni \\ --register-node=true \\ --v=2 Restart=on-failure @@ -288,6 +281,12 @@ EOF } ``` +Check if all Worker Services are active and are running +``` +sudo systemctl status containerd kubelet kube-proxy +``` + + > Remember to run the above commands on each worker node: `worker-0`, `worker-1`, and `worker-2`. ## Verification @@ -305,9 +304,9 @@ gcloud compute ssh controller-0 \ ``` NAME STATUS ROLES AGE VERSION -worker-0 Ready 22s v1.21.0 -worker-1 Ready 22s v1.21.0 -worker-2 Ready 22s v1.21.0 +worker-0 Ready 22s v1.26.1 +worker-1 Ready 22s v1.26.1 +worker-2 Ready 22s v1.26.1 ``` Next: [Configuring kubectl for Remote Access](10-configuring-kubectl.md) diff --git a/docs/10-configuring-kubectl.md b/docs/10-configuring-kubectl.md index 601fe28..421d0ed 100644 --- a/docs/10-configuring-kubectl.md +++ b/docs/10-configuring-kubectl.md @@ -44,8 +44,8 @@ kubectl version > output ``` -Client Version: version.Info{Major:"1", Minor:"21", GitVersion:"v1.21.0", GitCommit:"cb303e613a121a29364f75cc67d3d580833a7479", GitTreeState:"clean", BuildDate:"2021-04-08T16:31:21Z", GoVersion:"go1.16.1", Compiler:"gc", Platform:"linux/amd64"} -Server Version: version.Info{Major:"1", Minor:"21", GitVersion:"v1.21.0", GitCommit:"cb303e613a121a29364f75cc67d3d580833a7479", GitTreeState:"clean", BuildDate:"2021-04-08T16:25:06Z", GoVersion:"go1.16.1", Compiler:"gc", Platform:"linux/amd64"} +Client Version: version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.1", GitCommit:"8f94681cd294aa8cfd3407b8191f6c70214973a4", GitTreeState:"clean", BuildDate:"2023-01-18T15:58:16Z", GoVersion:"go1.19.5", Compiler:"gc", Platform:"linux/amd64"} +Server Version: version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.1", GitCommit:"8f94681cd294aa8cfd3407b8191f6c70214973a4", GitTreeState:"clean", BuildDate:"2023-01-18T15:51:25Z", GoVersion:"go1.19.5", Compiler:"gc", Platform:"linux/amd64"} ``` List the nodes in the remote Kubernetes cluster: @@ -58,9 +58,9 @@ kubectl get nodes ``` NAME STATUS ROLES AGE VERSION -worker-0 Ready 2m35s v1.21.0 -worker-1 Ready 2m35s v1.21.0 -worker-2 Ready 2m35s v1.21.0 +worker-0 Ready 2m35s v1.26.1 +worker-1 Ready 2m35s v1.26.1 +worker-2 Ready 2m35s v1.26.1 ``` Next: [Provisioning Pod Network Routes](11-pod-network-routes.md) diff --git a/docs/12-dns-addon.md b/docs/12-dns-addon.md index be81ef6..c1320d6 100644 --- a/docs/12-dns-addon.md +++ b/docs/12-dns-addon.md @@ -7,7 +7,7 @@ In this lab you will deploy the [DNS add-on](https://kubernetes.io/docs/concepts Deploy the `coredns` cluster add-on: ``` -kubectl apply -f https://storage.googleapis.com/kubernetes-the-hard-way/coredns-1.8.yaml +kubectl apply -f ./deployments/coredns-1.9.3.yaml ``` > output diff --git a/docs/13-smoke-test.md b/docs/13-smoke-test.md index 566ace7..65dfba7 100644 --- a/docs/13-smoke-test.md +++ b/docs/13-smoke-test.md @@ -111,13 +111,13 @@ curl --head http://127.0.0.1:8080 ``` HTTP/1.1 200 OK -Server: nginx/1.19.10 -Date: Sun, 02 May 2021 05:29:25 GMT +Server: nginx/1.23.3 +Date: Thu, 02 Feb 2023 12:52:56 GMT Content-Type: text/html -Content-Length: 612 -Last-Modified: Tue, 13 Apr 2021 15:13:59 GMT +Content-Length: 615 +Last-Modified: Tue, 13 Dec 2022 15:53:53 GMT Connection: keep-alive -ETag: "6075b537-264" +ETag: "6398a011-267" Accept-Ranges: bytes ``` @@ -160,7 +160,7 @@ kubectl exec -ti $POD_NAME -- nginx -v > output ``` -nginx version: nginx/1.19.10 +nginx version: nginx/1.23.3 ``` ## Services @@ -207,13 +207,13 @@ curl -I http://${EXTERNAL_IP}:${NODE_PORT} ``` HTTP/1.1 200 OK -Server: nginx/1.19.10 -Date: Sun, 02 May 2021 05:31:52 GMT +Server: nginx/1.23.3 +Date: Thu, 02 Feb 2023 12:56:09 GMT Content-Type: text/html -Content-Length: 612 -Last-Modified: Tue, 13 Apr 2021 15:13:59 GMT +Content-Length: 615 +Last-Modified: Tue, 13 Dec 2022 15:53:53 GMT Connection: keep-alive -ETag: "6075b537-264" +ETag: "6398a011-267" Accept-Ranges: bytes ```