Staging deployment (WIP)

staging/terraform/terraform.tfvars

+# These need to be defined for things to work
+ip_pool  = "PSNC-PUB1-EDU"
+net_name = "local-network"
+site_name = "staging"
+
+# These may need some adjustment for your provider
+master_cpus = 4
+master_ram  = 8192
+worker_cpus = 4
+worker_ram  = 8192
+# XXX: replace this for GPU flavor, once available
+gpu_flavor_name = "M1-NVME-2vCPU-8R-50D"
+
+# Number of extra workers
+extra_workers = 1
+
+# Number of GPU workers
+gpu_workers = 0
+
+# volumes for docker
+docker_volumes_size = 384
+
+# NFS volume
+nfs_volume_size = 256
+
+# scratch volume
+scratch_volumes_size = 128
+
+# squid volume
+squid_volume_size = 128

staging/terraform/vars.tf

+variable "ip_pool" {
+  type        = string
+  description = "The name of the public IP pool for the servers"
+}
+
+variable "net_name" {
+  type        = string
+  description = "The name of the IPv4 network"
+}
+
+variable "site_name" {
+  type        = string
+  description = "Site identifier for internal host names"
+}
+
+variable "gpu_flavor_name" {
+  type = string
+  description = "Name of the GPU flavor"
+}
+
+variable "master_cpus" {
+  type        = number
+  description = "Number of CPUs for the master"
+}
+
+variable "master_ram" {
+  type        = number
+  description = "RAM for the master"
+}
+
+variable "worker_cpus" {
+  type        = number
+  description = "Number of CPUs for the worker"
+}
+
+variable "worker_ram" {
+  type        = number
+  description = "RAM for the worker"
+}
+
+variable "extra_workers" {
+  type        = number
+  description = "Number of extra workers to create"
+}
+
+variable "gpu_workers" {
+  type        = number
+  description = "Number of GPU workers to create"
+}
+
+variable "docker_volumes_size" {
+  type        = number
+  description = "Size of volumes for docker (GB)"
+}
+
+variable "nfs_volume_size" {
+  type        = number
+  description = "Size of volume for NFS server (GB)"
+}
+
+variable "scratch_volumes_size" {
+  type        = number
+  description = "Size of volume for ephemeral volumes (GB)"
+}
+
+variable "squid_volume_size" {
+  type        = number
+  description = "Size of volume for squid proxy, CVMFS cache (GB)"
+}

staging/terraform/versions.tf

+terraform {
+  required_providers {
+    local = "~> 2.0"
+    openstack = {
+      source  = "terraform-provider-openstack/openstack",
+      version = ">= 1.38.0"
+    }
+  }
+  required_version = ">= 0.13"
+}

staging/terraform/vms.tf

+provider "openstack" {
+}
+
+locals {
+  nodes = concat([
+    openstack_compute_instance_v2.ingress,
+    openstack_compute_instance_v2.nfs,
+  ], openstack_compute_instance_v2.worker[*], openstack_compute_instance_v2.gpu[*])
+  master_ip = openstack_compute_instance_v2.master.network[0].fixed_ip_v4
+  ingress_ip = openstack_compute_instance_v2.ingress.network[0].fixed_ip_v4
+  nfs_ip = openstack_compute_instance_v2.nfs.network[0].fixed_ip_v4
+  worker_ips = [for s in openstack_compute_instance_v2.worker[*].network[0].fixed_ip_v4 : s]
+  gpu_ips = [for s in openstack_compute_instance_v2.gpu[*].network[0].fixed_ip_v4 : s]
+}
+
+# Security groups
+
+resource "openstack_networking_secgroup_v2" "ping" {
+  name        = "ping"
+  description = "ICMP for ping"
+}
+
+resource "openstack_networking_secgroup_v2" "ssh" {
+  name        = "ssh"
+  description = "ssh connection"
+}
+
+resource "openstack_networking_secgroup_v2" "http" {
+  name        = "http"
+  description = "http/https"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "ping4" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  port_range_min    = 8
+  port_range_max    = 0
+  protocol          = "icmp"
+  remote_ip_prefix  = "0.0.0.0/0"
+  security_group_id = openstack_networking_secgroup_v2.ping.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "ssh4" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  port_range_min    = 22
+  port_range_max    = 22
+  protocol          = "tcp"
+  remote_ip_prefix  = "0.0.0.0/0"
+  security_group_id = openstack_networking_secgroup_v2.ssh.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "ssh6" {
+  direction         = "ingress"
+  ethertype         = "IPv6"
+  port_range_min    = 22
+  port_range_max    = 22
+  protocol          = "tcp"
+  remote_ip_prefix  = "::/0"
+  security_group_id = openstack_networking_secgroup_v2.ssh.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "http4" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  port_range_min    = 80
+  port_range_max    = 80
+  protocol          = "tcp"
+  remote_ip_prefix  = "0.0.0.0/0"
+  security_group_id = openstack_networking_secgroup_v2.http.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "http6" {
+  direction         = "ingress"
+  ethertype         = "IPv6"
+  port_range_min    = 80
+  port_range_max    = 80
+  protocol          = "tcp"
+  remote_ip_prefix  = "::/0"
+  security_group_id = openstack_networking_secgroup_v2.http.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "https4" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  port_range_min    = 443
+  port_range_max    = 443
+  protocol          = "tcp"
+  remote_ip_prefix  = "0.0.0.0/0"
+  security_group_id = openstack_networking_secgroup_v2.http.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "https6" {
+  direction         = "ingress"
+  ethertype         = "IPv6"
+  port_range_min    = 443
+  port_range_max    = 443
+  protocol          = "tcp"
+  remote_ip_prefix  = "::/0"
+  security_group_id = openstack_networking_secgroup_v2.http.id
+}
+
+resource "openstack_networking_floatingip_v2" "public_ip" {
+  pool = var.ip_pool
+}
+
+data "openstack_images_image_v2" "ubuntu" {
+  name = "ubuntu-22.04-x86_64-server-cloudimg-20240429"
+}
+
+data "openstack_compute_flavor_v2" "master-flavor" {
+  vcpus = var.master_cpus
+  ram   = var.master_ram
+}
+
+data "openstack_compute_flavor_v2" "worker-flavor" {
+  vcpus = var.worker_cpus
+  ram   = var.worker_ram
+}
+
+data "openstack_compute_flavor_v2" "gpu-flavor" {
+  name = var.gpu_flavor_name
+}
+
+resource "openstack_compute_instance_v2" "master" {
+  name     = "k8s-${var.site_name}-master"
+  image_id = data.openstack_images_image_v2.ubuntu.id
+  flavor_id       = data.openstack_compute_flavor_v2.master-flavor.id
+  security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+  user_data       = file("cloud-init.yaml")
+  tags            = ["master"]
+  network {
+    name = var.net_name
+  }
+}
+
+resource "openstack_compute_instance_v2" "nfs" {
+  name            = "k8s-${var.site_name}-nfs"
+  image_id        = data.openstack_images_image_v2.ubuntu.id
+  flavor_id       = data.openstack_compute_flavor_v2.worker-flavor.id
+  security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+  user_data       = file("cloud-init.yaml")
+  tags            = ["worker"]
+  network {
+    name = var.net_name
+  }
+}
+
+resource "openstack_compute_instance_v2" "ingress" {
+  name            = "k8s-${var.site_name}-w-ingress"
+  image_id        = data.openstack_images_image_v2.ubuntu.id
+  flavor_id       = data.openstack_compute_flavor_v2.worker-flavor.id
+  security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name, openstack_networking_secgroup_v2.http.name]
+  user_data       = file("cloud-init.yaml")
+  tags            = ["worker"]
+  network {
+    name = var.net_name
+  }
+}
+
+resource "openstack_compute_instance_v2" "worker" {
+  count           = var.extra_workers
+  name            = "k8s-${var.site_name}-worker-${count.index}"
+  image_id        = data.openstack_images_image_v2.ubuntu.id
+  flavor_id       = data.openstack_compute_flavor_v2.worker-flavor.id
+  security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+  user_data       = file("cloud-init.yaml")
+  tags            = ["worker"]
+  network {
+    name = var.net_name
+  }
+}
+
+resource "openstack_compute_instance_v2" "gpu" {
+  count           = var.gpu_workers
+  name            = "k8s-${var.site_name}-gpu-${count.index}"
+  image_id        = data.openstack_images_image_v2.ubuntu.id
+  flavor_id       = data.openstack_compute_flavor_v2.gpu-flavor.id
+  security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+  user_data       = file("cloud-init.yaml")
+  tags = ["worker"]
+  network {
+    name = var.net_name
+  }
+}
+
+resource "openstack_compute_floatingip_associate_v2" "fip" {
+  floating_ip = openstack_networking_floatingip_v2.public_ip.address
+  instance_id = openstack_compute_instance_v2.ingress.id
+}
+
+resource "openstack_blockstorage_volume_v3" "nfs-volume" {
+  name = "nfs"
+  size = var.nfs_volume_size
+}
+
+resource "openstack_compute_volume_attach_v2" "nfs-volume-attach" {
+  instance_id = openstack_compute_instance_v2.nfs.id
+  volume_id   = openstack_blockstorage_volume_v3.nfs-volume.id
+}
+
+resource "local_file" "volume-script" {
+  filename        = "nfs-volume.sh"
+  file_permission = "0755"
+  content         = <<EOT
+#! /bin/bash -xe
+if ! dpkg-query -s xfsprogs >/dev/null 2>&1; then
+	apt-get update
+	apt-get install -y xfsprogs
+fi
+device="${openstack_compute_volume_attach_v2.nfs-volume-attach.device}"
+mkfs.xfs -L NFS "$device" || true
+grep -q 'LABEL=NFS' /etc/fstab || /bin/echo -e "LABEL=NFS\t/exports\txfs\tdefaults,uquota,pquota\t0\t0" | tee -a /etc/fstab
+mkdir /exports 2>/dev/null || true
+mount -a
+EOT
+}
+
+resource "openstack_blockstorage_volume_v3" "docker-volume" {
+  count = var.extra_workers + var.gpu_workers + 2
+  name  = format("docker-%s", local.nodes[count.index].name)
+  size  = var.docker_volumes_size
+}
+
+resource "openstack_compute_volume_attach_v2" "docker-volume-attach" {
+  count       = var.extra_workers + var.gpu_workers + 2
+  instance_id = local.nodes[count.index].id
+  volume_id   = openstack_blockstorage_volume_v3.docker-volume[count.index].id
+}
+
+resource "local_file" "docker-volume-script" {
+  filename        = "docker-volume.sh"
+  file_permission = "0755"
+  content         = <<EOT
+#! /bin/bash -xe
+volumes="${join("\n", [for n, d in zipmap(tolist(local.nodes[*].name), tolist(openstack_compute_volume_attach_v2.docker-volume-attach[*].device)) : format("%s:%s", n, d)])}"
+volume=$(echo "$volumes" | grep "$(hostname):")
+device=$(echo "$volume" | cut -d: -f2)
+if ! dumpe2fs -h "$device" >/dev/null 2>&1; then
+	mkfs.ext4 -L DOCKER "$device"
+	grep -q 'LABEL=DOCKER' /etc/fstab || /bin/echo -e "LABEL=DOCKER\t/var/lib/docker/overlay2\text4\tdefaults,x-systemd.before=local-fs.target\t0\t0" | tee -a /etc/fstab
+	mkdir -p /var/lib/docker/overlay2 2>/dev/null || true
+	service docker stop >/dev/null 2>&1 || true
+	sleep 10
+	umount /var/lib/docker/overlay2 2>&1 || true
+	mount "$device" /mnt
+	mv /var/lib/docker/overlay2/* /mnt >/dev/null 2>&1 || true
+	umount /mnt
+	mount -a
+fi
+EOT
+}
+
+resource "openstack_blockstorage_volume_v3" "scratch-volume" {
+  count = var.extra_workers + var.gpu_workers + 2
+  name  = format("scratch-%s", local.nodes[count.index].name)
+  size  = var.scratch_volumes_size
+}
+
+resource "openstack_compute_volume_attach_v2" "scratch-volume-attach" {
+  count       = var.extra_workers + var.gpu_workers + 2
+  instance_id = local.nodes[count.index].id
+  volume_id   = openstack_blockstorage_volume_v3.scratch-volume[count.index].id
+}
+
+resource "local_file" "scratch-volume-script" {
+  filename        = "scratch-volume.sh"
+  file_permission = "0755"
+  content         = <<EOT
+#! /bin/bash -xe
+volumes="${join("\n", [for n, d in zipmap(tolist(local.nodes[*].name), tolist(openstack_compute_volume_attach_v2.scratch-volume-attach[*].device)) : format("%s:%s", n, d)])}"
+volume=$(echo "$volumes" | grep "$(hostname):")
+device=$(echo "$volume" | cut -d: -f2)
+if ! dumpe2fs -h "$device" >/dev/null 2>&1; then
+	mkfs.ext4 -L SCRATCH "$device"
+	grep -q 'LABEL=SCRATCH' /etc/fstab || /bin/echo -e "LABEL=SCRATCH\t/scratch\text4\tdefaults\t0\t0" | tee -a /etc/fstab
+	mkdir -p /scratch 2>/dev/null || true
+	mount -a
+fi
+EOT
+}
+
+resource "openstack_blockstorage_volume_v3" "squid-volume" {
+  name = "squid"
+  size = var.squid_volume_size
+}
+
+resource "openstack_compute_volume_attach_v2" "squid-volume-attach" {
+  instance_id = openstack_compute_instance_v2.ingress.id
+  volume_id   = openstack_blockstorage_volume_v3.squid-volume.id
+}
+
+resource "local_file" "squid-volume-script" {
+  filename        = "squid-volume.sh"
+  file_permission = "0755"
+  content         = <<EOT
+#! /bin/bash -xe
+device="${openstack_compute_volume_attach_v2.squid-volume-attach.device}"
+if ! dumpe2fs -h "$device" >/dev/null 2>&1; then
+	mkfs.ext4 -L SQUID "$device"
+fi
+grep -q 'LABEL=SQUID' /etc/fstab || /bin/echo -e "LABEL=SQUID\t/var/spool/squid\text4\tdefaults,x-systemd.before=local-fs.target\t0\t0" | tee -a /etc/fstab
+mkdir /var/spool/squid 2>/dev/null || true
+mount -a
+EOT
+}
+
+resource "local_file" "inventory" {
+  filename        = "inventory.yaml"
+  file_permission = "0644"
+  content         = <<EOT
+---
+fip:
+  hosts:
+    ${openstack_networking_floatingip_v2.public_ip.address}:
+
+master:
+  hosts:
+    ${local.master_ip}:
+      # must be IPv4 address or hostname
+      kube_server: ${openstack_compute_instance_v2.master.network[0].fixed_ip_v4}
+
+ingress:
+  hosts:
+    ${local.ingress_ip}:
+
+nfs:
+  hosts:
+    ${local.nfs_ip}:
+
+worker:
+  hosts:
+    ${join("\n    ", [for s in local.worker_ips: "${s}:"])}
+
+gpu:
+  hosts:
+    ${join("\n    ", [for s in local.gpu_ips : "${s}:"])}
+
+# using public IP of kube_server for ansible delegate_to
+kube_server:
+  hosts:
+    ${openstack_compute_instance_v2.master.network[0].fixed_ip_v4}:
+      ansible_host: ${local.master_ip}
+EOT
+}
+
+resource "local_file" "fip" {
+  filename        = "fip.txt"
+  file_permission = "0644"
+  content         = <<EOT
+${openstack_networking_floatingip_v2.public_ip.address}
+EOT
+}
+
+resource "local_file" "hosts" {
+  filename        = "hosts.txt"
+  file_permission = "0644"
+  content         = <<EOT
+${local.master_ip}
+${local.ingress_ip}
+${local.nfs_ip}
+${join("\n", concat(local.worker_ips, local.gpu_ips))}
+EOT
+}