From aaa136f4878e0a8a0cf345505bea210d18b7c6a6 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Thu, 29 Sep 2016 18:22:00 +0900
Subject: [PATCH 01/14] infrastructure-azure

---
 .gitignore                      |   2 +
 docs/01-infrastructure-azure.md | 612 ++++++++++++++++++++++++++++++++
 2 files changed, 614 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 docs/01-infrastructure-azure.md

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..608d4bd
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+# Keys Directory
+keys
\ No newline at end of file
diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
new file mode 100644
index 0000000..e6745a7
--- /dev/null
+++ b/docs/01-infrastructure-azure.md
@@ -0,0 +1,612 @@
+# Cloud Infrastructure Provisioning - Azure
+
+## Variables
+
+```
+#change the following values as needed. 
+
+# dns for jumpbox is <jumpboxDnsLabel>.westus.cloudapp.azure.com
+jumpboxDnsLabel="the-hardway-way-jumpbox" 
+
+# dns for workers is <workersDnsLabel>.westus.cloudapp.azure.com
+workersDnsLabel="the-hardway-way" 
+
+#storage account used by jumpbox + controllers + Etcd VMs
+controlPlaneStorageAccount="thehardwaycsa" 
+
+#storage account used by workers
+workersStorageAccount="thehardwaywsa"
+
+# all vms are using ubunut 16.4 LTS 
+imageUrn="Canonical:UbuntuServer:16.04.0-LTS:latest"
+
+```
+
+## Create Resource Group
+
+```
+azure group create \
+	--name the-hard-way \
+	--location "West Us"
+```
+
+## Networking
+
+### Create Routing Table
+
+```
+azure network route-table create \
+	--resource-group the-hard-way \
+	--name the-hard-way-rtable \
+	--location "West Us"
+```
+
+### Create Network Security Group
+
+```
+azure network nsg create \
+	--resource-group the-hard-way \
+	--name the-hard-way-nsg \
+	--location "West Us"
+```
+
+
+Create NSG Rule Allowing SSH to Our Jump Box
+
+```
+azure network nsg rule create \
+	--resource-group the-hard-way \
+	--nsg-name the-hard-way-nsg \
+	--name allow-ssh-jumpbox \
+	--protocol tcp \
+	--access allow  \
+	--destination-address-prefix 10.0.0.4/32 \
+	--destination-port-range 22 \
+	--priority 100 \
+	--direction inbound
+```
+
+
+### Create VNET + Subnet 
+
+Cluster VNET
+```
+azure network vnet create \
+	--resource-group the-hard-way \
+	--name the-hard-way-net \
+	--address-prefixes 10.0.0.0/8 \
+	--location "West Us"
+```
+
+Create Kubernetes Subnet
+
+```
+azure network vnet subnet create \
+	--resource-group the-hard-way \
+	--vnet-name the-hard-way-net \
+	--name kubernetes \
+	--address-prefix 10.0.0.0/8 
+```
+
+Link Routing Table and NSG to Kubernetes Subnet
+
+```
+azure network vnet subnet set \
+	--resource-group the-hard-way \
+	--vnet-name the-hard-way-net \
+	--name kubernetes \
+	--network-security-group-name the-hard-way-nsg \
+	--route-table-name the-hard-way-rtable 
+```
+
+
+Create Public IP + DNS Lable for JumpBox
+
+```
+azure network public-ip create \
+	--resource-group the-hard-way \
+	--name the-hard-way-jumpbox  \
+	--allocation-method Static \
+	--domain-name-label $jumpboxDnsLabel \
+	--location "West Us"
+```
+
+## Virtual Machines
+
+Create SSH Key (Used by All VMs)
+
+```
+mkdir keys
+ssh-keygen -t rsa -f ./keys/cluster
+```
+
+### Storage Accounts 
+
+Create storage account for control plane VMs
+
+```
+azure storage account create $controlPlaneStorageAccount \
+	--resource-group the-hard-way \
+	--kind storage \
+	--sku-name LRS \
+	--location "West Us"
+```
+
+Create storage account for works VMs
+
+```
+azure storage account create $workersStorageAccount \
+	--resource-group the-hard-way \
+	--kind storage \
+	--sku-name LRS \
+	--location "West Us"
+```
+
+
+
+### Jump Box
+
+#### Create Nic (Private IP + Public IP)
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name jumpbox-nic \
+	--private-ip-address "10.0.0.4" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--public-ip-name the-hard-way-jumpbox \
+	--location "West Us"
+```
+
+#### Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+	--name jumpbox \
+	--vm-size Standard_A1 \
+	--nic-name jumpbox-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+	--os-type linux \
+	--image-urn $imageUrn \
+	--storage-account-name $controlPlaneStorageAccount \
+	--storage-account-container-name vhds \
+	--os-disk-vhd jumpbox.vhd \
+	--admin-username thehardway \
+	--ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+``` 
+
+### Etcd 
+
+#### Etcd 0
+
+Create Nic
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name etcd-0-nic \
+	--private-ip-address "10.240.0.10" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--location "West Us"  
+```
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+	--name etcd-0 \
+	--vm-size Standard_D4 \
+	--nic-name etcd-0-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+	--os-type linux \
+	--image-urn $imageUrn \
+	--storage-account-name $controlPlaneStorageAccount \
+	--storage-account-container-name vhds \
+	--os-disk-vhd etcd-0.vhd \
+	--admin-username thehardway \
+	--ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+#### Etcd 1
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name etcd-1-nic \
+	--private-ip-address "10.240.0.11" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--location "West Us"
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+	--name etcd-1 \
+	--vm-size Standard_D4 \
+	--nic-name etcd-1-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+	--os-type linux \
+	--image-urn $imageUrn \
+	--storage-account-name $controlPlaneStorageAccount \
+	--storage-account-container-name vhds \
+	--os-disk-vhd etcd-1.vhd \
+	--admin-username thehardway \
+	--ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+#### Etcd 2
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name etcd-2-nic \
+	--private-ip-address "10.240.0.12" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--location "West Us"  
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+	--name etcd-2 \
+	--vm-size Standard_D4 \
+	--nic-name etcd-2-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+	--os-type linux \
+	--image-urn $imageUrn \
+	--storage-account-name $controlPlaneStorageAccount \
+	--storage-account-container-name vhds \
+	--os-disk-vhd etcd-2.vhd \
+	--admin-username thehardway \
+	--ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+
+### Kubernetes Controllers
+
+#### Controller 0 
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name controller-0-nic \
+	--private-ip-address "10.240.0.20" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--location "West Us"
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+    --name controller-0 \
+    --vm-size Standard_D4 \
+    --nic-name controller-0-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+    --os-type linux \
+    --image-urn $imageUrn \
+    --storage-account-name $controlPlaneStorageAccount \
+    --storage-account-container-name vhds \
+    --os-disk-vhd controller-0.vhd \
+    --admin-username thehardway \
+    --ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+#### Controller 1 
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name controller-1-nic \
+	--private-ip-address "10.240.0.21" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--location "West Us"  
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+    --name controller-1 \
+    --vm-size Standard_D4 \
+    --nic-name controller-1-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+    --os-type linux \
+    --image-urn $imageUrn \
+    --storage-account-name $controlPlaneStorageAccount \
+    --storage-account-container-name vhds \
+    --os-disk-vhd controller-1.vhd \
+    --admin-username thehardway \
+    --ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+#### Controller 2
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name controller-2-nic \
+	--private-ip-address "10.240.0.22" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--location "West Us"  
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+    --name controller-2 \
+    --vm-size Standard_D4 \
+    --nic-names controller-2-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+    --os-type linux \
+    --image-urn $imageUrn \
+    --storage-account-name $controlPlaneStorageAccount \
+    --storage-account-container-name vhds \
+    --os-disk-vhd controller-2.vhd \
+    --admin-username thehardway \
+    --ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+
+### Kubernetes Workers
+
+#### Load Balancer 
+
+Create public IP + DNS label for workers ingestion load balancer
+
+```
+azure network public-ip create \
+	--resource-group the-hard-way \
+	--name the-hard-way-workers  \
+	--allocation-method Static \
+	--domain-name-label $workersDnsLabel \
+	--location "West Us"
+```
+
+Create load balancer 
+
+```
+azure network lb create \
+	--resource-group the-hard-way \
+	--name the-hard-way-lb \
+	--location "West Us"
+```
+
+Create & the front-end IP to the load balancer
+
+```
+azure network lb frontend-ip create \
+	--resource-group the-hard-way \
+	--name the-hard-way-fe  \
+	--lb-name the-hard-way-lb \
+	--public-ip-name the-hard-way-workers \
+	--subnet-vnet-name the-hard-way-net \
+	--subnet-name kubernetes
+```
+
+Create a backend address pool for the load balancer
+
+```
+backendPoolId=$(azure network lb address-pool create \
+	--resource-group the-hard-way \
+	--lb-name the-hard-way-lb \
+	--name backend-pool \
+	--json | jq -r '.id')
+```
+
+#### Create Workers Availablity set
+
+```
+azure availset create \
+	--resource-group the-hard-way \
+	--name workers-availset \
+	--location "West Us"
+```
+
+#### Worker 0
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name worker-0-nic \
+	--private-ip-address "10.240.0.30" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--enable-ip-forwarding "true" \
+	--lb-address-pool-ids $backendPoolId \
+	--location "West Us"  
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+    --name worker-0 \
+    --vm-size Standard_D4 \
+    --nic-name worker-0-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+    --availset-name workers-availset \
+	--os-type linux \
+    --image-urn $imageUrn \
+    --storage-account-name $workersStorageAccount \
+    --storage-account-container-name vhds \
+    --os-disk-vhd worker-0.vhd \
+    --admin-username thehardway \
+    --ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+#### Worker 1
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name worker-1-nic \
+	--private-ip-address "10.240.0.31" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--enable-ip-forwarding "true" \
+	--lb-address-pool-ids $backendPoolId \
+	--location "West Us"
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+    --name worker-1 \
+    --vm-size Standard_D4 \
+    --nic-name worker-1-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+    --availset-name workers-availset \
+	--os-type linux \
+    --image-urn $imageUrn \
+    --storage-account-name $workersStorageAccount \
+    --storage-account-container-name vhds \
+    --os-disk-vhd worker-1.vhd \
+    --admin-username thehardway \
+    --ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+#### Worker 2
+
+Create Nic
+
+```
+azure network nic create \
+	--resource-group the-hard-way \
+	--name worker-2-nic \
+	--private-ip-address "10.240.0.32" \
+	--subnet-vnet-name the-hard-way-net  \
+	--subnet-name kubernetes  \
+	--enable-ip-forwarding "true" \
+	--lb-address-pool-ids $backendPoolId \
+	--location "West Us" 
+```
+
+Create VM
+
+```
+azure vm create \
+	--resource-group the-hard-way \
+    --name worker-2 \
+    --vm-size Standard_D4 \
+    --nic-name worker-2-nic \
+	--vnet-name the-hard-way-net  \
+	--vnet-subnet-name kubernetes  \
+    --availset-name workers-availset \
+	--os-type linux \
+    --image-urn $imageUrn \
+    --storage-account-name $workersStorageAccount \
+    --storage-account-container-name vhds \
+    --os-disk-vhd worker-2.vhd \
+    --admin-username thehardway \
+    --ssh-publickey-file ./keys/cluster.pub \
+	--location "West US"
+```
+
+## Verify 
+
+```
+azure vm list --resource-group the-hard-way
+```
+
+Expected Output 
+```
+info:    Executing command vm list
++ Getting virtual machines                                                     
+data:    ResourceGroupName  Name          ProvisioningState  PowerState  Location  Size       
+data:    -----------------  ------------  -----------------  ----------  --------  -----------
+data:    the-hard-way       controller-0  Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       controller-1  Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       controller-2  Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       etcd-0        Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       etcd-1        Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       etcd-2        Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       jumpbox       Succeeded          VM running  westus    Standard_A1
+data:    the-hard-way       worker-0      Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       worker-1      Succeeded          VM running  westus    Standard_D4
+data:    the-hard-way       worker-2      Succeeded          VM running  westus    Standard_D4
+info:    vm list command OK
+```
+
+
+## Using The Jumpbox
+
+### Connect to Jumpbox 
+
+```
+ssh -i ./keys/cluster \
+	thehardway@$jumpboxDnsLabel.westus.cloudapp.azure.com
+```
+
+### Copy the Private Key to Jumpbox 
+
+```
+scp -i ./keys/cluster \
+	./keys/cluster \
+	thehardway@$jumpboxDnsLabel.westus.cloudapp.azure.com:~/cluster
+```
+
+### Connecting to Other VMs
+
+```
+# on the jumpbox 
+#connect to the second controller
+
+ssh -i ./cluster \
+	thehardway@10.240.0.31  
+
+#or
+ssh -i ./cluster \
+	thehardway@controller-1  
+
+```

From c0d89731d43d80eca4deb4393eeb2dddbe408756 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Fri, 30 Sep 2016 06:49:08 +0900
Subject: [PATCH 02/14] add internal load balancer for controlelrs

---
 docs/01-infrastructure-azure.md | 71 ++++++++++++++++++++++++++++-----
 1 file changed, 61 insertions(+), 10 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index e6745a7..8e31a58 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -122,7 +122,7 @@ ssh-keygen -t rsa -f ./keys/cluster
 
 ### Storage Accounts 
 
-Create storage account for control plane VMs
+Create storage account for control plane VMs (Etcd & Controllers)
 
 ```
 azure storage account create $controlPlaneStorageAccount \
@@ -152,7 +152,7 @@ azure storage account create $workersStorageAccount \
 azure network nic create \
 	--resource-group the-hard-way \
 	--name jumpbox-nic \
-	--private-ip-address "10.0.0.4" \
+	--private-ip-address "10.0.0.5" \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
 	--public-ip-name the-hard-way-jumpbox \
@@ -284,6 +284,51 @@ azure vm create \
 
 ### Kubernetes Controllers
 
+
+#### Workers Internal Load Balancer 
+
+
+Create load balancer 
+
+```
+azure network lb create \
+	--resource-group the-hard-way \
+	--name the-hard-way-clb \
+	--location "West Us"
+```
+
+Create & the front-end IP to the internal load balancer
+
+```
+azure network lb frontend-ip create \
+	--resource-group the-hard-way \
+	--name the-hard-way-cfe  \
+	--lb-name the-hard-way-clb \
+	--private-ip-address "10.0.0.4" \
+	--subnet-vnet-name the-hard-way-net \
+	--subnet-name kubernetes
+```
+
+Create a backend address pool for the load balancer
+
+```
+clbbackendPoolId=$(azure network lb address-pool create \
+	--resource-group the-hard-way \
+	--lb-name the-hard-way-clb \
+	--name backend-pool \
+	--json | jq -r '.id')
+```
+
+#### Create Controllers Availablity set
+
+```
+azure availset create \
+	--resource-group the-hard-way \
+	--name controllers-availset \
+	--location "West Us"
+```
+
+
 #### Controller 0 
 
 Create Nic
@@ -295,6 +340,7 @@ azure network nic create \
 	--private-ip-address "10.240.0.20" \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
+	--lb-address-pool-ids $clbbackendPoolId \
 	--location "West Us"
 ```
 
@@ -308,6 +354,7 @@ azure vm create \
     --nic-name controller-0-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
+	--availset-name controllers-availset \
     --os-type linux \
     --image-urn $imageUrn \
     --storage-account-name $controlPlaneStorageAccount \
@@ -329,7 +376,8 @@ azure network nic create \
 	--private-ip-address "10.240.0.21" \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
-	--location "West Us"  
+	--lb-address-pool-ids $clbbackendPoolId \
+	--location "West Us"
 ```
 
 Create VM
@@ -342,6 +390,7 @@ azure vm create \
     --nic-name controller-1-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
+	--availset-name controllers-availset \
     --os-type linux \
     --image-urn $imageUrn \
     --storage-account-name $controlPlaneStorageAccount \
@@ -363,7 +412,8 @@ azure network nic create \
 	--private-ip-address "10.240.0.22" \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
-	--location "West Us"  
+	--lb-address-pool-ids $clbbackendPoolId \
+	--location "West Us"
 ```
 
 Create VM
@@ -376,6 +426,7 @@ azure vm create \
     --nic-names controller-2-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
+	--availset-name controllers-availset \
     --os-type linux \
     --image-urn $imageUrn \
     --storage-account-name $controlPlaneStorageAccount \
@@ -389,7 +440,7 @@ azure vm create \
 
 ### Kubernetes Workers
 
-#### Load Balancer 
+#### Workers External Load Balancer 
 
 Create public IP + DNS label for workers ingestion load balancer
 
@@ -426,7 +477,7 @@ azure network lb frontend-ip create \
 Create a backend address pool for the load balancer
 
 ```
-backendPoolId=$(azure network lb address-pool create \
+wlbbackendPoolId=$(azure network lb address-pool create \
 	--resource-group the-hard-way \
 	--lb-name the-hard-way-lb \
 	--name backend-pool \
@@ -454,7 +505,7 @@ azure network nic create \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
 	--enable-ip-forwarding "true" \
-	--lb-address-pool-ids $backendPoolId \
+	--lb-address-pool-ids $wlbbackendPoolId \
 	--location "West Us"  
 ```
 
@@ -491,7 +542,7 @@ azure network nic create \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
 	--enable-ip-forwarding "true" \
-	--lb-address-pool-ids $backendPoolId \
+	--lb-address-pool-ids $wlbbackendPoolId \
 	--location "West Us"
 ```
 
@@ -528,8 +579,8 @@ azure network nic create \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
 	--enable-ip-forwarding "true" \
-	--lb-address-pool-ids $backendPoolId \
-	--location "West Us" 
+	--lb-address-pool-ids $wlbbackendPoolId \
+	--location "West Us"
 ```
 
 Create VM

From 6b711eedb7d3d10ea8037909758d30ee5e37da2e Mon Sep 17 00:00:00 2001
From: Jason Poon <git@jasonpoon.ca>
Date: Thu, 29 Sep 2016 15:26:47 -0700
Subject: [PATCH 03/14] Add link to azure infrastructure

---
 docs/01-infrastructure.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/01-infrastructure.md b/docs/01-infrastructure.md
index bd4ae69..a186ab3 100644
--- a/docs/01-infrastructure.md
+++ b/docs/01-infrastructure.md
@@ -6,3 +6,5 @@ This lab will walk you through provisioning the compute instances required for r
 
 * [Cloud Infrastructure Provisioning - Google Cloud Platform](01-infrastructure-gcp.md)
 * [Cloud Infrastructure Provisioning - Amazon Web Services](01-infrastructure-aws.md)
+* [Cloud Infrastructure Provisioning - Microsoft Azure](01-infrastructure-azure.md)
+

From a9b535f3a44da9238dd29cb8dd8a727458534aeb Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Fri, 30 Sep 2016 08:29:52 +0900
Subject: [PATCH 04/14] updated 01-infrastructure.md

---
 docs/01-infrastructure-azure.md | 16 ++++++++++++++--
 docs/01-infrastructure.md       |  4 +++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 8e31a58..7696fb0 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -1,4 +1,16 @@
 # Cloud Infrastructure Provisioning - Azure
+This lab will walk you through provisioning the compute instances required for running a H/A Kubernetes cluster. A total of 10 virtual machines will be created.
+
+The guide assumes you'll be creating resources in the `West Us` region as a single Azure Resource Manager resource group.
+
+After completing this guide you should have the following compute instances:
+
+##### add screen shot ####
+
+> All machines and load balancers will be provisioned with fixed private IP addresses to simplify the bootstrap process.
+
+The control plane machines are only accessible via a jump box (a VM with publically accessable ssh). The workers machines are exposed via external load balancer that carries both an public IP and public addressable dns FQDN. 
+
 
 ## Variables
 
@@ -6,10 +18,10 @@
 #change the following values as needed. 
 
 # dns for jumpbox is <jumpboxDnsLabel>.westus.cloudapp.azure.com
-jumpboxDnsLabel="the-hardway-way-jumpbox" 
+jumpboxDnsLabel="the-hard-way-jumpbox" 
 
 # dns for workers is <workersDnsLabel>.westus.cloudapp.azure.com
-workersDnsLabel="the-hardway-way" 
+workersDnsLabel="the-hard-way" 
 
 #storage account used by jumpbox + controllers + Etcd VMs
 controlPlaneStorageAccount="thehardwaycsa" 
diff --git a/docs/01-infrastructure.md b/docs/01-infrastructure.md
index bd4ae69..ef465cd 100644
--- a/docs/01-infrastructure.md
+++ b/docs/01-infrastructure.md
@@ -1,8 +1,10 @@
 # Cloud Infrastructure Provisioning
 
-Kubernetes can be installed just about anywhere physical or virtual machines can be run. In this lab we are going to focus on [Google Cloud Platform](https://cloud.google.com/) and [Amazon Web Services](https://aws.amazon.com).
+Kubernetes can be installed just about anywhere physical or virtual machines can be run. In this lab we are going to focus on [Google Cloud Platform](https://cloud.google.com/), [Amazon Web Services](https://aws.amazon.com) and [Microsoft Azure](https://azure.microsoft.com).
 
 This lab will walk you through provisioning the compute instances required for running a H/A Kubernetes cluster. 
 
 * [Cloud Infrastructure Provisioning - Google Cloud Platform](01-infrastructure-gcp.md)
 * [Cloud Infrastructure Provisioning - Amazon Web Services](01-infrastructure-aws.md)
+* [Cloud Infrastructure Provisioning - Microsoft Azure](01-infrastructure-azure.md)
+

From e0a3247ee5969ca7e28662dbefc3ada7c71afe57 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Fri, 30 Sep 2016 09:52:48 +0900
Subject: [PATCH 05/14] Fixed ssh port/ip in nsg

---
 docs/01-infrastructure-azure.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 7696fb0..08c894e 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -26,7 +26,7 @@ workersDnsLabel="the-hard-way"
 #storage account used by jumpbox + controllers + Etcd VMs
 controlPlaneStorageAccount="thehardwaycsa" 
 
-#storage account used by workers
+#storage account used by workers VMs
 workersStorageAccount="thehardwaywsa"
 
 # all vms are using ubunut 16.4 LTS 
@@ -72,7 +72,7 @@ azure network nsg rule create \
 	--name allow-ssh-jumpbox \
 	--protocol tcp \
 	--access allow  \
-	--destination-address-prefix 10.0.0.4/32 \
+	--destination-address-prefix 10.0.0.5/32 \
 	--destination-port-range 22 \
 	--priority 100 \
 	--direction inbound
@@ -97,7 +97,7 @@ azure network vnet subnet create \
 	--resource-group the-hard-way \
 	--vnet-name the-hard-way-net \
 	--name kubernetes \
-	--address-prefix 10.0.0.0/8 
+	--address-prefix 10.0.0.0/8
 ```
 
 Link Routing Table and NSG to Kubernetes Subnet
@@ -203,7 +203,7 @@ azure network nic create \
 	--private-ip-address "10.240.0.10" \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
-	--location "West Us"  
+	--location "West Us"
 ```
 Create VM
 
@@ -270,7 +270,7 @@ azure network nic create \
 	--private-ip-address "10.240.0.12" \
 	--subnet-vnet-name the-hard-way-net  \
 	--subnet-name kubernetes  \
-	--location "West Us"  
+	--location "West Us"
 ```
 
 Create VM
@@ -518,7 +518,7 @@ azure network nic create \
 	--subnet-name kubernetes  \
 	--enable-ip-forwarding "true" \
 	--lb-address-pool-ids $wlbbackendPoolId \
-	--location "West Us"  
+	--location "West Us"
 ```
 
 Create VM

From ca0913f4f30fcf91cff2f9c0ff91dd4334b7e2a0 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Fri, 30 Sep 2016 11:04:01 +0900
Subject: [PATCH 06/14] fixed host name to match existing doc +  Modified CA
 instruction for Azure

---
 docs/01-infrastructure-azure.md  | 18 ++++++------
 docs/02-certificate-authority.md | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 08c894e..95fbaf1 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -210,7 +210,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-	--name etcd-0 \
+	--name etcd0 \
 	--vm-size Standard_D4 \
 	--nic-name etcd-0-nic \
 	--vnet-name the-hard-way-net  \
@@ -244,7 +244,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-	--name etcd-1 \
+	--name etcd1 \
 	--vm-size Standard_D4 \
 	--nic-name etcd-1-nic \
 	--vnet-name the-hard-way-net  \
@@ -278,7 +278,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-	--name etcd-2 \
+	--name etcd2 \
 	--vm-size Standard_D4 \
 	--nic-name etcd-2-nic \
 	--vnet-name the-hard-way-net  \
@@ -361,7 +361,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-    --name controller-0 \
+    --name controller0 \
     --vm-size Standard_D4 \
     --nic-name controller-0-nic \
 	--vnet-name the-hard-way-net  \
@@ -397,7 +397,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-    --name controller-1 \
+    --name controller1 \
     --vm-size Standard_D4 \
     --nic-name controller-1-nic \
 	--vnet-name the-hard-way-net  \
@@ -433,7 +433,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-    --name controller-2 \
+    --name controller2 \
     --vm-size Standard_D4 \
     --nic-names controller-2-nic \
 	--vnet-name the-hard-way-net  \
@@ -526,7 +526,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-    --name worker-0 \
+    --name worker0 \
     --vm-size Standard_D4 \
     --nic-name worker-0-nic \
 	--vnet-name the-hard-way-net  \
@@ -563,7 +563,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-    --name worker-1 \
+    --name worker1 \
     --vm-size Standard_D4 \
     --nic-name worker-1-nic \
 	--vnet-name the-hard-way-net  \
@@ -600,7 +600,7 @@ Create VM
 ```
 azure vm create \
 	--resource-group the-hard-way \
-    --name worker-2 \
+    --name worker2 \
     --vm-size Standard_D4 \
     --nic-name worker-2-nic \
 	--vnet-name the-hard-way-net  \
diff --git a/docs/02-certificate-authority.md b/docs/02-certificate-authority.md
index 2ccaf36..300febe 100644
--- a/docs/02-certificate-authority.md
+++ b/docs/02-certificate-authority.md
@@ -137,6 +137,14 @@ KUBERNETES_PUBLIC_ADDRESS=$(aws elb describe-load-balancers \
   jq -r '.LoadBalancerDescriptions[].DNSName')
 ```
 
+#### Azure
+```
+KUBERNETES_PUBLIC_ADDRESS=$(azure network public-ip show \
+  --resource-group the-hard-way \
+  --name the-hard-way-workers \
+  --json | jq -r '.dnsSettings.fqdn')
+```
+
 ---
 
 Create the `kubernetes-csr.json` file:
@@ -242,3 +250,42 @@ for host in ${KUBERNETES_HOSTS[*]}; do
     ubuntu@${PUBLIC_IP_ADDRESS}:~/
 done
 ```
+
+### Azure
+If you used the jumpbox to configure the CA
+```
+for host in ${KUBERNETES_HOSTS[*]}; do
+  scp -i ./cluster ca.pem kubernetes-key.pem kubernetes.pem \
+    thehardway@${host}:~/
+done
+```
+If you used a different machine
+```
+
+#Get jumpbox address
+
+KUBERNETES_JUMPBOX_ADDRESS=$(azure network public-ip show \
+  --resource-group the-hard-way \
+  --name the-hard-way-jumpbox \
+  --json | jq -r '.dnsSettings.fqdn')
+
+#Copy files to jumpbox 
+scp -i ./keys/cluster \
+  ca.pem \
+  kubernetes-key.pem \
+  kubernetes.pem \
+  thehardway@$KUBERNETES_JUMPBOX_ADDRESS:~/
+
+# Copy files from jumpbox to vms
+ssh -i ./keys/cluster \
+  thehardway@$KUBERNETES_JUMPBOX_ADDRESS <<'EOF'
+  
+  KUBERNETES_HOSTS=(controller0 controller1 controller2 etcd0 etcd1 etcd2 worker0 worker1 worker2)
+  for host in ${KUBERNETES_HOSTS[*]}; do
+  scp -i ./cluster ca.pem kubernetes-key.pem kubernetes.pem \
+    thehardway@${host}:~/
+  done
+  
+EOF
+
+```
\ No newline at end of file

From e42c2b4aae37dc0c5907b2a70c18fcdce30aec01 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Fri, 30 Sep 2016 12:44:38 -0700
Subject: [PATCH 07/14] completed etcd config for Azure + fixes on
 infrastructure-azure file

---
 docs/01-infrastructure-azure.md  | 18 +++++++++---------
 docs/02-certificate-authority.md |  9 +++++----
 docs/03-etcd.md                  |  4 ++++
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 95fbaf1..4de5138 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -24,7 +24,7 @@ jumpboxDnsLabel="the-hard-way-jumpbox"
 workersDnsLabel="the-hard-way" 
 
 #storage account used by jumpbox + controllers + Etcd VMs
-controlPlaneStorageAccount="thehardwaycsa" 
+controlPlaneStorageAccount="thehardwaycsa"
 
 #storage account used by workers VMs
 workersStorageAccount="thehardwaywsa"
@@ -108,11 +108,11 @@ azure network vnet subnet set \
 	--vnet-name the-hard-way-net \
 	--name kubernetes \
 	--network-security-group-name the-hard-way-nsg \
-	--route-table-name the-hard-way-rtable 
+	--route-table-name the-hard-way-rtable
 ```
 
 
-Create Public IP + DNS Lable for JumpBox
+Create Public IP + DNS label for JumpBox
 
 ```
 azure network public-ip create \
@@ -297,10 +297,10 @@ azure vm create \
 ### Kubernetes Controllers
 
 
-#### Workers Internal Load Balancer 
+#### Controllers Internal Load Balancer 
 
 
-Create load balancer 
+Create controllers load balancer 
 
 ```
 azure network lb create \
@@ -309,7 +309,7 @@ azure network lb create \
 	--location "West Us"
 ```
 
-Create & the front-end IP to the internal load balancer
+Create & assign the front-end private IP to the internal load balancer
 
 ```
 azure network lb frontend-ip create \
@@ -331,7 +331,7 @@ clbbackendPoolId=$(azure network lb address-pool create \
 	--json | jq -r '.id')
 ```
 
-#### Create Controllers Availablity set
+#### Create controllers availability set
 
 ```
 azure availset create \
@@ -474,7 +474,7 @@ azure network lb create \
 	--location "West Us"
 ```
 
-Create & the front-end IP to the load balancer
+Assign the front-end public IP to the load balancer
 
 ```
 azure network lb frontend-ip create \
@@ -651,7 +651,7 @@ ssh -i ./keys/cluster \
 	thehardway@$jumpboxDnsLabel.westus.cloudapp.azure.com
 ```
 
-### Copy the Private Key to Jumpbox 
+### Copy the cluster private key to Jumpbox 
 
 ```
 scp -i ./keys/cluster \
diff --git a/docs/02-certificate-authority.md b/docs/02-certificate-authority.md
index 300febe..dde628e 100644
--- a/docs/02-certificate-authority.md
+++ b/docs/02-certificate-authority.md
@@ -139,10 +139,11 @@ KUBERNETES_PUBLIC_ADDRESS=$(aws elb describe-load-balancers \
 
 #### Azure
 ```
-KUBERNETES_PUBLIC_ADDRESS=$(azure network public-ip show \
+KUBERNETES_PUBLIC_ADDRESS=$(azure network lb show \
   --resource-group the-hard-way \
-  --name the-hard-way-workers \
-  --json | jq -r '.dnsSettings.fqdn')
+  --name the-hard-way-clb \
+  --json | \
+  jq -r '.frontendIPConfigurations[0].privateIPAddress')
 ```
 
 ---
@@ -252,7 +253,7 @@ done
 ```
 
 ### Azure
-If you used the jumpbox to configure the CA
+If you are using the jumpbox to create the certificates
 ```
 for host in ${KUBERNETES_HOSTS[*]}; do
   scp -i ./cluster ca.pem kubernetes-key.pem kubernetes.pem \
diff --git a/docs/03-etcd.md b/docs/03-etcd.md
index 069f7a1..f2eae9d 100644
--- a/docs/03-etcd.md
+++ b/docs/03-etcd.md
@@ -95,6 +95,10 @@ INTERNAL_IP=$(curl -s -H "Metadata-Flavor: Google" \
 INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 ```
 
+#### Azure
+```
+INTERNAL_IP=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{print $1}')
+```
 ---
 
 Set the etcd name:

From 918e0169cadcc7d901c67c9be27cea37f5448822 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Sat, 1 Oct 2016 10:05:30 -0700
Subject: [PATCH 08/14] completed: controllers, workers + kubectl

---
 docs/04-kubernetes-controller.md | 29 +++++++++++++++++++++++++++++
 docs/06-kubectl.md               | 14 ++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/docs/04-kubernetes-controller.md b/docs/04-kubernetes-controller.md
index 2807470..489d57b 100644
--- a/docs/04-kubernetes-controller.md
+++ b/docs/04-kubernetes-controller.md
@@ -125,6 +125,11 @@ INTERNAL_IP=$(curl -s -H "Metadata-Flavor: Google" \
 INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 ```
 
+#### Azure
+```
+INTERNAL_IP=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{print $1}')
+```
+
 ---
 
 Create the systemd unit file:
@@ -328,3 +333,27 @@ aws elb register-instances-with-load-balancer \
   --load-balancer-name kubernetes \
   --instances ${CONTROLLER_0_INSTANCE_ID} ${CONTROLLER_1_INSTANCE_ID} ${CONTROLLER_2_INSTANCE_ID}
 ```
+
+
+### Azure
+```
+azure network lb probe create \
+  --resource-group the-hard-way \
+  --lb-name the-hard-way-clb \
+  --name controller-api-server-health \
+  --interval 5 \
+  --port 8080 \
+  --protocol http \
+  --path '/healthz'
+
+azure network lb rule create \
+  --resource-group the-hard-way \
+  --lb-name the-hard-way-clb \
+  --name controller-api-server \
+  --frontend-port 6443 \
+  --backend-port 6443 \
+  --frontend-ip-name the-hard-way-cfe \
+  --backend-address-pool-name backend-pool\
+  --probe-name controller-api-server-health
+
+```
\ No newline at end of file
diff --git a/docs/06-kubectl.md b/docs/06-kubectl.md
index a8ed209..c4a8b4e 100644
--- a/docs/06-kubectl.md
+++ b/docs/06-kubectl.md
@@ -36,6 +36,20 @@ KUBERNETES_PUBLIC_ADDRESS=$(aws elb describe-load-balancers \
   --load-balancer-name kubernetes | \
   jq -r '.LoadBalancerDescriptions[].DNSName')
 ```
+
+### Azure 
+
+```
+# for this work, we are configuring kubectl on jumpbox
+# The controllers are exposed via internal load balancer
+# access is only allowed within the VNET 
+# (or ssh -L ... port 6443 .. from jumpbox to internal lb) 
+KUBERNETES_PUBLIC_ADDRESS=$(azure network lb show \
+  --resource-group the-hard-way \
+  --name the-hard-way-clb \
+  --json | \
+  jq -r '.frontendIPConfigurations[0].privateIPAddress')
+```
 ---
 
 Recall the token we setup for the admin user:

From c31b40de8b990c4b547530d5f6a853440a498e86 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Sat, 1 Oct 2016 15:40:07 -0700
Subject: [PATCH 09/14] split subnets to allow UDR to work + completed all
 configs

---
 docs/01-infrastructure-azure.md  | 49 ++++++++++++++++++++++----------
 docs/02-certificate-authority.md |  7 +++++
 docs/03-etcd.md                  |  2 ++
 docs/04-kubernetes-controller.md |  3 +-
 docs/07-network.md               | 28 ++++++++++++++++++
 docs/09-smoke-test.md            | 46 ++++++++++++++++++++++++++++++
 docs/10-cleanup.md               |  8 ++++++
 7 files changed, 127 insertions(+), 16 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 4de5138..fee2b27 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -90,19 +90,40 @@ azure network vnet create \
 	--location "West Us"
 ```
 
-Create Kubernetes Subnet
+Create Subnets
 
 ```
+# Azure UDR routes traffic going outside 
+# the subnet
+# workers have to be on their own subnet
+
+azure network vnet subnet create \
+	--resource-group the-hard-way \
+	--vnet-name the-hard-way-net \
+	--name kubernetes-mgmt \
+	--address-prefix 10.0.0.0/16
+
+
 azure network vnet subnet create \
 	--resource-group the-hard-way \
 	--vnet-name the-hard-way-net \
 	--name kubernetes \
-	--address-prefix 10.0.0.0/8
-```
+	--address-prefix 10.239.0.0/11
 
-Link Routing Table and NSG to Kubernetes Subnet
 
 ```
+
+Link routing table and NSG to Kubernetes/-mgmt subnets
+
+```
+azure network vnet subnet set \
+	--resource-group the-hard-way \
+	--vnet-name the-hard-way-net \
+	--name kubernetes-mgmt \
+	--network-security-group-name the-hard-way-nsg \
+	--route-table-name the-hard-way-rtable
+
+
 azure network vnet subnet set \
 	--resource-group the-hard-way \
 	--vnet-name the-hard-way-net \
@@ -112,13 +133,13 @@ azure network vnet subnet set \
 ```
 
 
-Create Public IP + DNS label for JumpBox
+Create public IP + DNS label for the jumpbox
 
 ```
 azure network public-ip create \
 	--resource-group the-hard-way \
 	--name the-hard-way-jumpbox  \
-	--allocation-method Static \
+	--allocation-method Dynamic \
 	--domain-name-label $jumpboxDnsLabel \
 	--location "West Us"
 ```
@@ -166,7 +187,7 @@ azure network nic create \
 	--name jumpbox-nic \
 	--private-ip-address "10.0.0.5" \
 	--subnet-vnet-name the-hard-way-net  \
-	--subnet-name kubernetes  \
+	--subnet-name kubernetes-mgmt  \
 	--public-ip-name the-hard-way-jumpbox \
 	--location "West Us"
 ```
@@ -180,7 +201,7 @@ azure vm create \
 	--vm-size Standard_A1 \
 	--nic-name jumpbox-nic \
 	--vnet-name the-hard-way-net  \
-	--vnet-subnet-name kubernetes  \
+	--vnet-subnet-name kubernetes-mgmt  \
 	--os-type linux \
 	--image-urn $imageUrn \
 	--storage-account-name $controlPlaneStorageAccount \
@@ -300,7 +321,7 @@ azure vm create \
 #### Controllers Internal Load Balancer 
 
 
-Create controllers load balancer 
+Create controllers internal load balancer 
 
 ```
 azure network lb create \
@@ -316,7 +337,7 @@ azure network lb frontend-ip create \
 	--resource-group the-hard-way \
 	--name the-hard-way-cfe  \
 	--lb-name the-hard-way-clb \
-	--private-ip-address "10.0.0.4" \
+	--private-ip-address "10.240.0.4" \
 	--subnet-vnet-name the-hard-way-net \
 	--subnet-name kubernetes
 ```
@@ -454,13 +475,13 @@ azure vm create \
 
 #### Workers External Load Balancer 
 
-Create public IP + DNS label for workers ingestion load balancer
+Create public IP + DNS label for workers ingestion external load balancer
 
 ```
 azure network public-ip create \
 	--resource-group the-hard-way \
 	--name the-hard-way-workers  \
-	--allocation-method Static \
+	--allocation-method Dynamic \
 	--domain-name-label $workersDnsLabel \
 	--location "West Us"
 ```
@@ -481,9 +502,7 @@ azure network lb frontend-ip create \
 	--resource-group the-hard-way \
 	--name the-hard-way-fe  \
 	--lb-name the-hard-way-lb \
-	--public-ip-name the-hard-way-workers \
-	--subnet-vnet-name the-hard-way-net \
-	--subnet-name kubernetes
+	--public-ip-name the-hard-way-workers
 ```
 
 Create a backend address pool for the load balancer
diff --git a/docs/02-certificate-authority.md b/docs/02-certificate-authority.md
index dde628e..5eb8500 100644
--- a/docs/02-certificate-authority.md
+++ b/docs/02-certificate-authority.md
@@ -138,6 +138,9 @@ KUBERNETES_PUBLIC_ADDRESS=$(aws elb describe-load-balancers \
 ```
 
 #### Azure
+
+this gets the address of the internal controllers load balancer
+
 ```
 KUBERNETES_PUBLIC_ADDRESS=$(azure network lb show \
   --resource-group the-hard-way \
@@ -253,14 +256,18 @@ done
 ```
 
 ### Azure
+
 If you are using the jumpbox to create the certificates
+
 ```
 for host in ${KUBERNETES_HOSTS[*]}; do
   scp -i ./cluster ca.pem kubernetes-key.pem kubernetes.pem \
     thehardway@${host}:~/
 done
 ```
+
 If you used a different machine
+
 ```
 
 #Get jumpbox address
diff --git a/docs/03-etcd.md b/docs/03-etcd.md
index f2eae9d..3fa1586 100644
--- a/docs/03-etcd.md
+++ b/docs/03-etcd.md
@@ -96,9 +96,11 @@ INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 ```
 
 #### Azure
+
 ```
 INTERNAL_IP=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{print $1}')
 ```
+
 ---
 
 Set the etcd name:
diff --git a/docs/04-kubernetes-controller.md b/docs/04-kubernetes-controller.md
index 489d57b..24aeda1 100644
--- a/docs/04-kubernetes-controller.md
+++ b/docs/04-kubernetes-controller.md
@@ -126,6 +126,7 @@ INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 ```
 
 #### Azure
+
 ```
 INTERNAL_IP=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{print $1}')
 ```
@@ -336,6 +337,7 @@ aws elb register-instances-with-load-balancer \
 
 
 ### Azure
+
 ```
 azure network lb probe create \
   --resource-group the-hard-way \
@@ -355,5 +357,4 @@ azure network lb rule create \
   --frontend-ip-name the-hard-way-cfe \
   --backend-address-pool-name backend-pool\
   --probe-name controller-api-server-health
-
 ```
\ No newline at end of file
diff --git a/docs/07-network.md b/docs/07-network.md
index 0d66381..3e6fe5c 100644
--- a/docs/07-network.md
+++ b/docs/07-network.md
@@ -118,3 +118,31 @@ aws ec2 create-route \
   --destination-cidr-block 10.200.2.0/24 \
   --instance-id ${WORKER_2_INSTANCE_ID}
 ```
+
+### Azure
+
+```
+azure network route-table route create \
+  --resource-group the-hard-way \
+  --name worker0-route \
+  --route-table-name the-hard-way-rtable \
+  --address-prefix 10.200.0.0/24 \
+  --next-hop-ip-address 10.240.0.30 \
+  --next-hop-type VirtualAppliance
+
+azure network route-table route create \
+  --resource-group the-hard-way \
+  --name worker1-route \
+  --route-table-name the-hard-way-rtable \
+  --address-prefix 10.200.1.0/24 \
+  --next-hop-ip-address 10.240.0.31 \
+  --next-hop-type VirtualAppliance
+
+azure network route-table route create \
+  --resource-group the-hard-way \
+  --name worker2-route \
+  --route-table-name the-hard-way-rtable \
+  --address-prefix 10.200.2.0/24 \
+  --next-hop-ip-address 10.240.0.32 \
+  --next-hop-type VirtualAppliance  
+```
\ No newline at end of file
diff --git a/docs/09-smoke-test.md b/docs/09-smoke-test.md
index a812108..3d8af30 100644
--- a/docs/09-smoke-test.md
+++ b/docs/09-smoke-test.md
@@ -79,6 +79,52 @@ NODE_PUBLIC_IP=$(aws ec2 describe-instances \
   jq -j '.Reservations[].Instances[].PublicIpAddress')
 ```
 
+#### Azure
+
+```
+# Get the fqdn for the public worker ingestion load balancer
+
+NODE_PUBLIC_IP=$(azure network public-ip show \
+  --resource-group the-hard-way \
+  --name the-hard-way-workers \
+  --json | jq -r '.dnsSettings.fqdn')
+
+# Add NSG rule to enable traffic to node ports
+
+azure network nsg rule create \
+	--resource-group the-hard-way \
+	--nsg-name the-hard-way-nsg \
+	--name allow-internet-$NODE_PORT \
+	--protocol tcp \
+	--access allow  \
+  --source-address-prefix Internet \
+	--destination-address-prefix 10.240.0.0/16 \
+	--destination-port-range $NODE_PORT \
+	--priority 110 \
+	--direction inbound
+
+# Create balancing rules NODE_PORT:NODE_PORT on the load balancer
+
+azure network lb probe create \
+  --resource-group the-hard-way \
+  --lb-name the-hard-way-lb \
+  --name nginx-app-health \
+  --interval 5 \
+  --port $NODE_PORT \
+  --protocol tcp
+  
+
+azure network lb rule create \
+  --resource-group the-hard-way \
+  --lb-name the-hard-way-lb \
+  --name nginx-app  \
+  --frontend-port $NODE_PORT \
+  --backend-port $NODE_PORT \
+  --frontend-ip-name the-hard-way-fe \
+  --backend-address-pool-name backend-pool\
+  --probe-name nginx-app-health
+```
+
 ---
 
 Test the nginx service using cURL:
diff --git a/docs/10-cleanup.md b/docs/10-cleanup.md
index d59006a..977641a 100644
--- a/docs/10-cleanup.md
+++ b/docs/10-cleanup.md
@@ -206,3 +206,11 @@ DHCP_OPTION_SET_ID=$(aws ec2 describe-dhcp-options \
 aws ec2 delete-dhcp-options \
   --dhcp-options-id ${DHCP_OPTION_SET_ID}
 ```
+
+## Azure 
+
+The following deletes all resources created.
+
+```
+azure group delete the-hard-way
+```
\ No newline at end of file

From 51235f4234a1dc99a98547495cf290057ba328fc Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Sat, 1 Oct 2016 16:09:16 -0700
Subject: [PATCH 10/14] doc review + updates

---
 README.md                        |  9 ++++++-
 docs/01-infrastructure-azure.md  | 42 +++++++++++++++++---------------
 docs/02-certificate-authority.md |  6 ++---
 docs/06-kubectl.md               |  4 +--
 docs/09-smoke-test.md            |  4 +--
 5 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/README.md b/README.md
index cbabade..8f7ccaa 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,8 @@ This tutorial is optimized for learning, which means taking the long route to he
 
 * [Google Compute Engine](https://cloud.google.com/compute)
 * [Amazon EC2](https://aws.amazon.com/ec2)
+* [Microsoft Azure](https://azure.microsoft.com)
+
 
 > The results of this tutorial should not be viewed as production ready, and may receive limited support from the community, but don't let that prevent you from learning!
 
@@ -46,16 +48,21 @@ AWS
 
 * The us-west-2 region will be used
 
+AWS
+
+* The "west us" region will be used
+
 ## Platforms
 
 This tutorial assumes you have access to one of the following:
 
 * [Google Cloud Platform](https://cloud.google.com) and the [Google Cloud SDK](https://cloud.google.com/sdk/) (125.0.0+)
 * [Amazon Web Services](https://aws.amazon.com), the [AWS CLI](https://aws.amazon.com/cli) (1.10.63+), and [jq](https://stedolan.github.io/jq) (1.5+)
+* [Microsoft Azure](https://azure.microsoft.com), the [Azure CLI](https://azure.microsoft.com/en-us/documentation/articles/xplat-cli-install/) (0.10.1+), and [jq](https://stedolan.github.io/jq) (1.5+)
 
 ## Labs
 
-While GCP or AWS will be used for basic infrastructure needs, the things learned in this tutorial apply to every platform.
+While GCP, AWS or Azure will be used for basic infrastructure needs, the things learned in this tutorial apply to every platform.
 
 * [Cloud Infrastructure Provisioning](docs/01-infrastructure.md)
 * [Setting up a CA and TLS Cert Generation](docs/02-certificate-authority.md)
diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index fee2b27..203849d 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -3,35 +3,39 @@ This lab will walk you through provisioning the compute instances required for r
 
 The guide assumes you'll be creating resources in the `West Us` region as a single Azure Resource Manager resource group.
 
-After completing this guide you should have the following compute instances:
 
-##### add screen shot ####
+> All machines will be provisioned with fixed private IP addresses to simplify the bootstrap process.
 
-> All machines and load balancers will be provisioned with fixed private IP addresses to simplify the bootstrap process.
-
-The control plane machines are only accessible via a jump box (a VM with publically accessable ssh). The workers machines are exposed via external load balancer that carries both an public IP and public addressable dns FQDN. 
+The cluster VNs are only accessible via a jump box (a VM with publicly accessible ssh endpoint). The workers machines are exposed via external load balancer that carries both an public IP and public FQDN. 
 
 
 ## Variables
 
-```
-#change the following values as needed. 
+Change the following values as needed. 
 
+```
 # dns for jumpbox is <jumpboxDnsLabel>.westus.cloudapp.azure.com
 jumpboxDnsLabel="the-hard-way-jumpbox" 
+```
 
+```
 # dns for workers is <workersDnsLabel>.westus.cloudapp.azure.com
 workersDnsLabel="the-hard-way" 
+```
 
+```
 #storage account used by jumpbox + controllers + Etcd VMs
 controlPlaneStorageAccount="thehardwaycsa"
+```
 
+```
 #storage account used by workers VMs
 workersStorageAccount="thehardwaywsa"
+```
 
+```
 # all vms are using ubunut 16.4 LTS 
 imageUrn="Canonical:UbuntuServer:16.04.0-LTS:latest"
-
 ```
 
 ## Create Resource Group
@@ -63,7 +67,7 @@ azure network nsg create \
 ```
 
 
-Create NSG Rule Allowing SSH to Our Jump Box
+Create NSG rule allowing SSH to the jumpbox
 
 ```
 azure network nsg rule create \
@@ -90,12 +94,11 @@ azure network vnet create \
 	--location "West Us"
 ```
 
-Create Subnets
+Create subnets
 
 ```
-# Azure UDR routes traffic going outside 
-# the subnet
-# workers have to be on their own subnet
+# Azure UDR routes traffic subnet's eggress
+# workers & pod ips have to be 2 separate subnets
 
 azure network vnet subnet create \
 	--resource-group the-hard-way \
@@ -146,7 +149,7 @@ azure network public-ip create \
 
 ## Virtual Machines
 
-Create SSH Key (Used by All VMs)
+Create SSH keys (Used by All VMs)
 
 ```
 mkdir keys
@@ -165,7 +168,7 @@ azure storage account create $controlPlaneStorageAccount \
 	--location "West Us"
 ```
 
-Create storage account for works VMs
+Create storage account for workers VMs
 
 ```
 azure storage account create $workersStorageAccount \
@@ -179,7 +182,7 @@ azure storage account create $workersStorageAccount \
 
 ### Jump Box
 
-#### Create Nic (Private IP + Public IP)
+#### Create Nic (Private IP + Public IP + FQDN)
 
 ```
 azure network nic create \
@@ -320,8 +323,7 @@ azure vm create \
 
 #### Controllers Internal Load Balancer 
 
-
-Create controllers internal load balancer 
+Create load balancer 
 
 ```
 azure network lb create \
@@ -495,7 +497,7 @@ azure network lb create \
 	--location "West Us"
 ```
 
-Assign the front-end public IP to the load balancer
+Assign the front-end public IP + FQDN to the load balancer
 
 ```
 azure network lb frontend-ip create \
@@ -670,7 +672,7 @@ ssh -i ./keys/cluster \
 	thehardway@$jumpboxDnsLabel.westus.cloudapp.azure.com
 ```
 
-### Copy the cluster private key to Jumpbox 
+### Copy the cluster private key to jumpbox 
 
 ```
 scp -i ./keys/cluster \
diff --git a/docs/02-certificate-authority.md b/docs/02-certificate-authority.md
index 5eb8500..50b5884 100644
--- a/docs/02-certificate-authority.md
+++ b/docs/02-certificate-authority.md
@@ -270,14 +270,15 @@ If you used a different machine
 
 ```
 
-#Get jumpbox address
+# Get jumpbox address
 
 KUBERNETES_JUMPBOX_ADDRESS=$(azure network public-ip show \
   --resource-group the-hard-way \
   --name the-hard-way-jumpbox \
   --json | jq -r '.dnsSettings.fqdn')
 
-#Copy files to jumpbox 
+# Copy files to jumpbox 
+
 scp -i ./keys/cluster \
   ca.pem \
   kubernetes-key.pem \
@@ -295,5 +296,4 @@ ssh -i ./keys/cluster \
   done
   
 EOF
-
 ```
\ No newline at end of file
diff --git a/docs/06-kubectl.md b/docs/06-kubectl.md
index c4a8b4e..f1e6a8a 100644
--- a/docs/06-kubectl.md
+++ b/docs/06-kubectl.md
@@ -40,10 +40,10 @@ KUBERNETES_PUBLIC_ADDRESS=$(aws elb describe-load-balancers \
 ### Azure 
 
 ```
-# for this work, we are configuring kubectl on jumpbox
+# we are configuring kubectl on jumpbox
 # The controllers are exposed via internal load balancer
 # access is only allowed within the VNET 
-# (or ssh -L ... port 6443 .. from jumpbox to internal lb) 
+# (outside the vnet ssh -L ... port 6443 .. from jumpbox to internal lb) 
 KUBERNETES_PUBLIC_ADDRESS=$(azure network lb show \
   --resource-group the-hard-way \
   --name the-hard-way-clb \
diff --git a/docs/09-smoke-test.md b/docs/09-smoke-test.md
index 3d8af30..7d90d1f 100644
--- a/docs/09-smoke-test.md
+++ b/docs/09-smoke-test.md
@@ -89,7 +89,7 @@ NODE_PUBLIC_IP=$(azure network public-ip show \
   --name the-hard-way-workers \
   --json | jq -r '.dnsSettings.fqdn')
 
-# Add NSG rule to enable traffic to node ports
+# Add NSG rule to enable traffic to workers' node ports
 
 azure network nsg rule create \
 	--resource-group the-hard-way \
@@ -103,7 +103,7 @@ azure network nsg rule create \
 	--priority 110 \
 	--direction inbound
 
-# Create balancing rules NODE_PORT:NODE_PORT on the load balancer
+# Create load balancer rule NODE_PORT:NODE_PORT on the load balancer
 
 azure network lb probe create \
   --resource-group the-hard-way \

From 70c2f85e8c728f582c700a1a66a2ba1f2490fb39 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Mon, 3 Oct 2016 18:20:15 -0700
Subject: [PATCH 11/14] fixed the kubernetes address space

---
 docs/01-infrastructure-azure.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 203849d..4cc7cc6 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -111,7 +111,7 @@ azure network vnet subnet create \
 	--resource-group the-hard-way \
 	--vnet-name the-hard-way-net \
 	--name kubernetes \
-	--address-prefix 10.239.0.0/11
+	--address-prefix 10.224.0.0/11
 
 
 ```

From 3e67675d508a3deea43c1f3e6fb28b00d01a8add Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Tue, 4 Oct 2016 11:32:01 -0700
Subject: [PATCH 12/14] Doc updates and typo fixes

---
 README.md                       | 2 +-
 docs/01-infrastructure-azure.md | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 8f7ccaa..8039af0 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ AWS
 
 * The us-west-2 region will be used
 
-AWS
+Azure
 
 * The "west us" region will be used
 
diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 4cc7cc6..e18da9f 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -6,7 +6,7 @@ The guide assumes you'll be creating resources in the `West Us` region as a sing
 
 > All machines will be provisioned with fixed private IP addresses to simplify the bootstrap process.
 
-The cluster VNs are only accessible via a jump box (a VM with publicly accessible ssh endpoint). The workers machines are exposed via external load balancer that carries both an public IP and public FQDN. 
+The cluster VMs are only accessible via a jump box (a VM with publicly accessible ssh endpoint). The workers machines are exposed via external load balancer that carries both an public IP and public FQDN. 
 
 
 ## Variables
@@ -97,8 +97,9 @@ azure network vnet create \
 Create subnets
 
 ```
-# Azure UDR routes traffic subnet's eggress
-# workers & pod ips have to be 2 separate subnets
+# Azure UDR "user defined routes" in custom routing tables 
+# routes traffic leaving the subnet.
+# Workers & pods (IPs) have to be in two separate subnets
 
 azure network vnet subnet create \
 	--resource-group the-hard-way \

From dfd38efca05a7286232fa62dbeead2b6e2bd279e Mon Sep 17 00:00:00 2001
From: Jason Poon <git@jasonpoon.ca>
Date: Tue, 4 Oct 2016 14:58:09 -0700
Subject: [PATCH 13/14] edits

---
 docs/01-infrastructure-azure.md | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index e18da9f..626bce7 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -57,7 +57,7 @@ azure network route-table create \
 	--location "West Us"
 ```
 
-### Create Network Security Group
+### Create Network Security Group (NSG)
 
 ```
 azure network nsg create \
@@ -76,7 +76,7 @@ azure network nsg rule create \
 	--name allow-ssh-jumpbox \
 	--protocol tcp \
 	--access allow  \
-	--destination-address-prefix 10.0.0.5/32 \
+	--destination-address-prefix 10.0.0.5 \
 	--destination-port-range 22 \
 	--priority 100 \
 	--direction inbound
@@ -117,7 +117,7 @@ azure network vnet subnet create \
 
 ```
 
-Link routing table and NSG to Kubernetes/-mgmt subnets
+Associate the routing table and NSG to Kubernetes/-mgmt subnets
 
 ```
 azure network vnet subnet set \
@@ -183,7 +183,7 @@ azure storage account create $workersStorageAccount \
 
 ### Jump Box
 
-#### Create Nic (Private IP + Public IP + FQDN)
+#### Create NIC (Private IP + Public IP + FQDN)
 
 ```
 azure network nic create \
@@ -236,7 +236,7 @@ Create VM
 azure vm create \
 	--resource-group the-hard-way \
 	--name etcd0 \
-	--vm-size Standard_D4 \
+	--vm-size Standard_A1 \
 	--nic-name etcd-0-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
@@ -270,7 +270,7 @@ Create VM
 azure vm create \
 	--resource-group the-hard-way \
 	--name etcd1 \
-	--vm-size Standard_D4 \
+	--vm-size Standard_A1 \
 	--nic-name etcd-1-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
@@ -304,7 +304,7 @@ Create VM
 azure vm create \
 	--resource-group the-hard-way \
 	--name etcd2 \
-	--vm-size Standard_D4 \
+	--vm-size Standard_A1 \
 	--nic-name etcd-2-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
@@ -386,7 +386,7 @@ Create VM
 azure vm create \
 	--resource-group the-hard-way \
     --name controller0 \
-    --vm-size Standard_D4 \
+    --vm-size Standard_A1 \
     --nic-name controller-0-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
@@ -422,7 +422,7 @@ Create VM
 azure vm create \
 	--resource-group the-hard-way \
     --name controller1 \
-    --vm-size Standard_D4 \
+    --vm-size Standard_A1 \
     --nic-name controller-1-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
@@ -458,7 +458,7 @@ Create VM
 azure vm create \
 	--resource-group the-hard-way \
     --name controller2 \
-    --vm-size Standard_D4 \
+    --vm-size Standard_A1 \
     --nic-names controller-2-nic \
 	--vnet-name the-hard-way-net  \
 	--vnet-subnet-name kubernetes  \
@@ -684,7 +684,7 @@ scp -i ./keys/cluster \
 ### Connecting to Other VMs
 
 ```
-# on the jumpbox 
+#from the jumpbox 
 #connect to the second controller
 
 ssh -i ./cluster \

From 9fd42fe1b607d85f76fe0b34b2cd3f4ce3648326 Mon Sep 17 00:00:00 2001
From: khenidak <khnidk@outlook.com>
Date: Wed, 5 Oct 2016 11:04:43 -0700
Subject: [PATCH 14/14] Review updates and fixes

---
 docs/01-infrastructure-azure.md  | 2 ++
 docs/04-kubernetes-controller.md | 2 +-
 docs/09-smoke-test.md            | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/01-infrastructure-azure.md b/docs/01-infrastructure-azure.md
index 626bce7..51aaf89 100644
--- a/docs/01-infrastructure-azure.md
+++ b/docs/01-infrastructure-azure.md
@@ -666,6 +666,8 @@ info:    vm list command OK
 
 ## Using The Jumpbox
 
+> The Jumpbox does not have Azure CLI installed. All further Azure CLI commands should be executed on a machine with CLI installed & configured.   
+
 ### Connect to Jumpbox 
 
 ```
diff --git a/docs/04-kubernetes-controller.md b/docs/04-kubernetes-controller.md
index 24aeda1..ced240c 100644
--- a/docs/04-kubernetes-controller.md
+++ b/docs/04-kubernetes-controller.md
@@ -355,6 +355,6 @@ azure network lb rule create \
   --frontend-port 6443 \
   --backend-port 6443 \
   --frontend-ip-name the-hard-way-cfe \
-  --backend-address-pool-name backend-pool\
+  --backend-address-pool-name backend-pool \
   --probe-name controller-api-server-health
 ```
\ No newline at end of file
diff --git a/docs/09-smoke-test.md b/docs/09-smoke-test.md
index 7d90d1f..3f486c4 100644
--- a/docs/09-smoke-test.md
+++ b/docs/09-smoke-test.md
@@ -121,7 +121,7 @@ azure network lb rule create \
   --frontend-port $NODE_PORT \
   --backend-port $NODE_PORT \
   --frontend-ip-name the-hard-way-fe \
-  --backend-address-pool-name backend-pool\
+  --backend-address-pool-name backend-pool \
   --probe-name nginx-app-health
 ```