Initial commit

build-cluster/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+inventory = ./inventory.yaml
+host_key_checking = False
+remote_user = kube
+roles_path = ./roles

build-cluster/config/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

build-cluster/install_k3s.yaml

@@ -0,0 +1,5 @@
+---
+- name: Install k3s
+  hosts: all
+  roles:
+    - k3s-cluster-installer 

build-cluster/inventory.yaml

@@ -0,0 +1,44 @@
+all:
+  vars:
+    cp_fqdn: nc_cp.home.thejimnicholson.com
+    proxmox_api: pve.home.thejimnicholson.com
+    pdns_api: 10.0.96.30
+    pdns_dom: home.thejimnicholson.com
+    cloudinit_img: centos-8-cloudimg
+  children:
+    primary:
+      hosts:
+        nc001:
+          mem: 4096
+          node: pve
+          id: 3001
+          ip: "10.0.96.111"
+    control_plane:
+      hosts:
+        nc002:
+          node: pve2
+          mem: 4096
+          id: 3002
+          ip: "10.0.96.112"
+        nc003:
+          node: pve3
+          mem: 4096
+          id: 3003
+          ip: "10.0.96.113"
+    workers:
+      hosts:
+        nc004:
+          node: pve
+          mem: 8192
+          id: 3004
+          ip: "10.0.96.114"
+        nc005:
+          node: pve2
+          mem: 8192
+          id: 3005
+          ip: "10.0.96.115"
+        nc006:
+          node: pve3
+          mem: 8192
+          id: 3006
+          ip: "10.0.96.116"

build-cluster/prepare-cluster-nodes.yaml

@@ -0,0 +1,47 @@
+- name: Prep cluster nodes
+  hosts:
+    - primary
+    - control_plane
+    - workers
+  tasks:
+    - name: Set hostname
+      ansible.builtin.hostname:
+        name: "{{ inventory_hostname }}"
+      become: true
+    - name: Set timezone
+      community.general.timezone:
+        name: America/Los_Angeles
+      become: true
+    - name: Disable swap
+      shell: swapoff -a
+      become: true
+    - name: Disable swap in fstab
+      replace:
+        path: /etc/fstab
+        regexp: '^([^#].*?\sswap\s+sw\s+.*)$'
+        replace: '# \1'
+      become: true
+    - name: Add GPG for elrepo
+      ansible.builtin.rpm_key:
+        state: present
+        key: https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
+      become: true
+    - name : Set up elrepo
+      dnf:
+        name: https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm
+        state: present
+      become: true
+    - name: Install kernel upgrade
+      dnf:
+        enablerepo: elrepo-kernel
+        name: kernel-ml
+        state: present
+      become: true
+    - name: Install iscsi drivers
+      dnf:
+        name: iscsi-initiator-utils
+        state: present
+      become: true
+    - name: Reboot servers
+      reboot:
+      become: true

build-cluster/provision-cluster-nodes.yaml

@@ -0,0 +1,85 @@
+---
+- name: Provision k3s cluster nodes
+  hosts:
+    - localhost
+  gather_facts: no
+  tasks:
+    - name: Clone host image
+      community.general.proxmox_kvm:
+        api_host: "{{ proxmox_api_host }}"
+        api_user: "{{ lookup('env','PM_USER') }}"
+        api_password: "{{ lookup('env','PM_PASSWORD') }}"        
+        autostart: true
+        clone: "{{ cloudinit_img }}"
+        name: "{{ item }}"
+        node: pve2
+        target: "{{ hostvars[item].node }}"
+        storage: disk-storage
+        format: qcow2
+        newid: "{{ hostvars[item].id }}"
+        timeout: 500
+      with_inventory_hostnames:
+        - primary
+        - control_plane
+        - workers        
+    - name: Update clones
+      community.general.proxmox_kvm:
+        update: yes
+        node: "{{ hostvars[item].node }}"
+        api_host: "{{ proxmox_api_host }}"
+        api_user: "{{ lookup('env','PM_USER') }}"
+        api_password: "{{ lookup('env','PM_PASSWORD') }}"      
+        memory: "{{ hostvars[item].mem }}"
+        ciuser: kube
+        cipassword: Call1_advent
+        cores: 4
+        sshkeys: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC0Lk7zShZhujYeAnqorxZZCUJbZWWzf74cyAXRCGFeqyUvzOHuc/M3s0dmEqWRQCnKKdRAcAeBuya8dNyXwlTbWGTMbrObesPb0rHgLLXUfPbDH1km9QpVufjpuhbUtBN0iSa/1n3vKeMvrQj3ekUvl6nRtcLPHY0H4RswOJSpEzpvHK8S4YxdSoBV0z9KVB3/nS45WsqY45pD75epEjgaEhxyiJkf2fy5VkEB0+ZRMWs4uv/emwXq1hparkh5618Qap5qTpxI0kG0gXjupYc9HYe3oqHtxXsqrN3G/wEX6bVsbxNUdU5WMlqT88TkbRcju7UI7UhNcBezXaeT/WlJZGGM2spEHpk7DBC8Td6t09vCDQzRU694p6/hMfOUS3aMSdcIIU9wOdqXaXgFW2ugUxjQV0L0EowCxX8wJpPmxECs+svf7cCPYZVyF+R4MKHHx2mW/GtHYceAkQvIMfjPg2ZlKNOWGAJHBjjnLlAdXWJf77+FH5q3QCQshEW4loe/7/cd3AUGplYtHKxBaGYJS8YRDDmAE/TBZsm3ICaGCIUtEEsIBrSPR+f6WFU5fMIOh82735FGMI1rO4rNkFJ3ZBgwsgurY1yKrtP8yrTKELF0noycWw6DYHrwShXDFKjIlut5w3L3tOdlL6gheTfZqhSwDNZXm/3H76BfOUpxEQ== jim@DESKTOP-PP2J3PP'
+        vmid: "{{ hostvars[item].id }}"
+        nameservers:
+          '10.0.96.30'
+        net:
+          net0: 'virtio,bridge=vmbr0'
+        ipconfig:
+          ipconfig0: "ip={{ hostvars[item].ip }}/24,gw=10.0.96.1"
+      with_inventory_hostnames:
+        - primary
+        - control_plane
+        - workers 
+    - name: Start clones
+      community.general.proxmox_kvm:
+        node: "{{ hostvars[item].node }}"
+        api_host: "{{ proxmox_api_host }}"
+        api_user: "{{ lookup('env','PM_USER') }}"
+        api_password: "{{ lookup('env','PM_PASSWORD') }}"         
+        update: yes
+        vmid: "{{ hostvars[item].id }}"
+        state: started
+      with_inventory_hostnames:
+        - primary
+        - control_plane
+        - workers 
+    - name: Update DNS for cluster nodes
+      uri:
+        method: PATCH
+        url: "http://{{ pdns_api }}:8081/api/v1/servers/localhost/zones/{{ pdns_dom }}"
+        body:
+          rrsets:
+            - name: "{{ item }}.{{ pdns_dom }}."
+              type: A
+              ttl: 86400
+              changetype: REPLACE
+              records:
+                - content: "{{ hostvars[item].ip }}"
+                  disabled: false
+        body_format: json
+        headers:
+          'X-API-Key': '{{ lookup('env','PDNS_API_PW') }}'
+        return_content: yes
+        status_code: 
+          - 200
+          - 204
+      register: dns_result
+      with_inventory_hostnames:
+        - primary
+        - control_plane
+        - workers

build-cluster/roles/k3s-cluster-installer/files/flannel.conf

@@ -0,0 +1,2 @@
+[keyfile]
+unmanaged-devices=interface-name:flannel*;interface-name:cni*;interface-name:tunl*;interface-name:vxlan.calico

build-cluster/roles/k3s-cluster-installer/tasks/control_plane.yaml

@@ -0,0 +1,23 @@
+- name: Install the rest of the control plane
+  block:
+    - name: Get the token
+      set_fact:
+        token: "{{ lookup('file',playbook_dir + '/config/token') }}"
+    - name: Get the primary ip
+      set_fact:
+        primary_ip: "{{ lookup('file',playbook_dir + '/config/primary_ip') }}"
+    - name: Run the installer
+      shell:
+        cmd: "curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='server --server https://{{ primary_ip }}:6443  --tls-san {{ cp_fqdn }} --write-kubeconfig-mode 644 --token {{ token }}' sh -"
+      args:
+        creates: /etc/rancher/k3s/k3s.yaml
+      register: installed_cp
+      throttle: 1
+    - name: Pause to let the rest of the control plane to come up
+      wait_for:
+        timeout: 30
+      become: false
+      delegate_to: localhost
+      when: installed_cp.changed
+  become: true
+  when: "'control_plane' in group_names"

build-cluster/roles/k3s-cluster-installer/tasks/main.yaml

@@ -0,0 +1,5 @@
+---
+- import_tasks: prep.yaml
+- import_tasks: primary.yaml
+- import_tasks: control_plane.yaml
+- import_tasks: workers.yaml

build-cluster/roles/k3s-cluster-installer/tasks/prep.yaml

@@ -0,0 +1,24 @@
+---
+- name: Create config directory
+  file:
+    path: /etc/rancher/k3s
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  become: true
+- name: Disable NetworkManager for container networks
+  copy:
+    src: flannel.conf
+    dest: /etc/NetworkManager/conf.d/flannel.conf
+    owner: root
+    group: root
+    mode: 0644
+  become: true
+  register: nm_update
+- name: Restart NetworkManager
+  systemd:
+    name: NetworkManager
+    state: restarted
+  become: true
+  when: nm_update.changed

build-cluster/roles/k3s-cluster-installer/tasks/primary.yaml

@@ -0,0 +1,39 @@
+- name: Install the primary node
+  block:
+    - name: Set the primary IP fact
+      set_fact:
+        primary_ip: "{{ ansible_default_ipv4.address }}"
+    - copy:
+        content={{ primary_ip }}
+        dest="{{ playbook_dir }}/config/primary_ip"
+      become: false
+      delegate_to: localhost
+    - name: Run the installer
+      shell:
+        cmd: "curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='server --cluster-init --tls-san {{ cp_fqdn }} --write-kubeconfig-mode 644' sh -"
+      args:
+        creates: /etc/rancher/k3s/k3s.yaml
+      register: installed_primary
+    - name: Download kubeconfig file
+      fetch:
+        src: /etc/rancher/k3s/k3s.yaml
+        dest: "{{ playbook_dir }}/config/"
+        flat: yes
+    - name: Download the node join token
+      fetch:
+        src: /var/lib/rancher/k3s/server/token
+        dest: "{{ playbook_dir }}/config/"
+        flat: yes
+    - name: Fix URL for control plane
+      shell:
+        cmd: sed -i.bak "s/127.0.0.1/{{ ansible_default_ipv4.address }}/g" {{playbook_dir}}/config/k3s.yaml
+      become: false
+      delegate_to: localhost
+    - name: Pause to let the service come up
+      wait_for:
+        timeout: 60
+      become: false
+      delegate_to: localhost
+      when: installed_primary.changed
+  become: true
+  when: "'primary' in group_names"

build-cluster/roles/k3s-cluster-installer/tasks/workers.yaml

@@ -0,0 +1,15 @@
+- name: Install the worker nodes
+  block:
+    - name: Get the token
+      set_fact:
+        token: "{{ lookup('file',playbook_dir + '/config/token') }}"
+    - name: Get the primary ip
+      set_fact:
+        primary_ip: "{{ lookup('file',playbook_dir + '/config/primary_ip') }}"
+    - name: Run the installer
+      shell:
+        cmd: "curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent --server https://{{ primary_ip }}:6443 --token {{ token }}' sh -"
+      args:
+        creates: /var/lib/rancher/k3s/agent/k3scontroller.kubeconfig 
+  become: true
+  when: "'workers' in group_names"