From 1a8f3ff2a05fe92bb764a84b58b233f9fae10143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Thu, 18 Jul 2024 16:52:46 +0000
Subject: [PATCH] Moar refactoring and cleanup
---
cesnet-mcc/terraform/terraform.tfvars | 13 +++--
cesnet-mcc/terraform/vars.tf | 60 +--------------------
cesnet-mcc/terraform/vms.tf | 39 +++++++++++---
staging1/terraform/terraform.tfvars | 8 +--
staging1/terraform/vars.tf | 65 +----------------------
staging1/terraform/vms.tf | 8 ++-
staging2/terraform/terraform.tfvars | 9 ++--
staging2/terraform/vars.tf | 65 +----------------------
staging2/terraform/vms.tf | 20 +++----
testing/terraform/terraform.tfvars | 6 +--
testing/terraform/vars.tf | 75 +--------------------------
testing/terraform/vms.tf | 10 ++--
12 files changed, 69 insertions(+), 309 deletions(-)
mode change 100644 => 120000 cesnet-mcc/terraform/vars.tf
mode change 100644 => 120000 staging1/terraform/vars.tf
mode change 100644 => 120000 staging2/terraform/vars.tf
mode change 100644 => 120000 testing/terraform/vars.tf
diff --git a/cesnet-mcc/terraform/terraform.tfvars b/cesnet-mcc/terraform/terraform.tfvars
index a6a30f4..041c17d 100644
--- a/cesnet-mcc/terraform/terraform.tfvars
+++ b/cesnet-mcc/terraform/terraform.tfvars
@@ -5,19 +5,24 @@ net6_name = "public-muni-v6-432"
site_name = "cesnet-mcc"
# These may need some adjustment for your provider
-master_cpus = 2 # 2 CPUs to match existing flavours
-master_ram = 4096
-worker_cpus = 4
-worker_ram = 8192
+master_flavor_name = "standard.medium"
+worker_flavor_name = "standard.large"
+gpu_flavor_name = "a3.32core-240ram-1t4"
# Number of extra workers
extra_workers = 2
+# Number of GPU workers
+gpu_workers = 0
+
# volumes for docker
docker_volumes_size = 384
# NFS volume
nfs_volume_size = 256
+# scratch volume
+scratch_volumes_size = 128
+
# squid volume
squid_volume_size = 128
diff --git a/cesnet-mcc/terraform/vars.tf b/cesnet-mcc/terraform/vars.tf
deleted file mode 100644
index 2e39ee9..0000000
--- a/cesnet-mcc/terraform/vars.tf
+++ /dev/null
@@ -1,59 +0,0 @@
-variable "ip_pool" {
- type = string
- description = "The name of the public IP pool for the servers"
-}
-
-variable "net_name" {
- type = string
- description = "The name of the IPv4 network"
-}
-
-variable "net6_name" {
- type = string
- description = "The name of the IPv6 network"
-}
-
-variable "site_name" {
- type = string
- description = "Site identifier for internal host names"
-}
-
-variable "master_cpus" {
- type = number
- description = "Number of CPUs for the master"
-}
-
-variable "master_ram" {
- type = number
- description = "RAM for the master"
-}
-
-variable "worker_cpus" {
- type = number
- description = "Number of CPUs for the worker"
-}
-
-variable "worker_ram" {
- type = number
- description = "RAM for the worker"
-}
-
-variable "extra_workers" {
- type = number
- description = "Number of extra workers to create"
-}
-
-variable "docker_volumes_size" {
- type = number
- description = "Size of volumes for docker (GB)"
-}
-
-variable "nfs_volume_size" {
- type = number
- description = "Size of volume for NFS server (GB)"
-}
-
-variable "squid_volume_size" {
- type = number
- description = "Size of volume for squid proxy, CVMFS cache (GB)"
-}
diff --git a/cesnet-mcc/terraform/vars.tf b/cesnet-mcc/terraform/vars.tf
new file mode 120000
index 0000000..00c4e3a
--- /dev/null
+++ b/cesnet-mcc/terraform/vars.tf
@@ -0,0 +1 @@
+../../common/terraform/vars.tf
\ No newline at end of file
diff --git a/cesnet-mcc/terraform/vms.tf b/cesnet-mcc/terraform/vms.tf
index deabb55..4d4275a 100644
--- a/cesnet-mcc/terraform/vms.tf
+++ b/cesnet-mcc/terraform/vms.tf
@@ -2,11 +2,12 @@ locals {
nodes = concat([
openstack_compute_instance_v2.ingress,
openstack_compute_instance_v2.nfs,
- ], openstack_compute_instance_v2.worker[*])
+ ], openstack_compute_instance_v2.worker[*], openstack_compute_instance_v2.gpu[*])
master_ip = replace(openstack_compute_instance_v2.master.network[1].fixed_ip_v6, "/\\[(.*)\\]/", "$1")
ingress_ip = replace(openstack_compute_instance_v2.ingress.network[1].fixed_ip_v6, "/\\[(.*)\\]/", "$1")
nfs_ip = replace(openstack_compute_instance_v2.nfs.network[1].fixed_ip_v6, "/\\[(.*)\\]/", "$1")
worker_ips = [for s in openstack_compute_instance_v2.worker[*].network[1].fixed_ip_v6 : replace(s, "/\\[(.*)\\]/", "$1")]
+ gpu_ips = [for s in openstack_compute_instance_v2.gpu[*].network[1].fixed_ip_v6 : replace(s, "/\\[(.*)\\]/", "$1")]
}
# Security groups
@@ -87,13 +88,15 @@ data "openstack_images_image_v2" "ubuntu" {
}
data "openstack_compute_flavor_v2" "master-flavor" {
- vcpus = var.master_cpus
- ram = var.master_ram
+ name = var.master_flavor_name
}
data "openstack_compute_flavor_v2" "worker-flavor" {
- vcpus = var.worker_cpus
- ram = var.worker_ram
+ name = var.worker_flavor_name
+}
+
+data "openstack_compute_flavor_v2" "gpu-flavor" {
+ name = var.gpu_flavor_name
}
resource "openstack_compute_instance_v2" "master" {
@@ -158,6 +161,22 @@ resource "openstack_compute_instance_v2" "worker" {
}
}
+resource "openstack_compute_instance_v2" "gpu" {
+ count = var.gpu_workers
+ name = "k8s-${var.site_name}-gpu-${count.index}"
+ image_id = data.openstack_images_image_v2.ubuntu.id
+ flavor_id = data.openstack_compute_flavor_v2.gpu-flavor.id
+ security_groups = ["default", openstack_compute_secgroup_v2.ping.name, openstack_compute_secgroup_v2.ssh.name]
+ user_data = file("cloud-init.yaml")
+ tags = ["worker"]
+ network {
+ name = var.net_name
+ }
+ network {
+ name = var.net6_name
+ }
+}
+
resource "openstack_compute_floatingip_associate_v2" "fip" {
floating_ip = openstack_networking_floatingip_v2.public_ip.address
instance_id = openstack_compute_instance_v2.ingress.id
@@ -191,13 +210,13 @@ EOT
}
resource "openstack_blockstorage_volume_v3" "docker-volume" {
- count = var.extra_workers + 2
+ count = var.extra_workers + var.gpu_workers + 2
name = format("docker-%s", local.nodes[count.index].name)
size = var.docker_volumes_size
}
resource "openstack_compute_volume_attach_v2" "docker-volume-attach" {
- count = var.extra_workers + 2
+ count = var.extra_workers + var.gpu_workers + 2
instance_id = local.nodes[count.index].id
volume_id = openstack_blockstorage_volume_v3.docker-volume[count.index].id
}
@@ -279,6 +298,10 @@ worker:
hosts:
${join("\n ", [for s in local.worker_ips: "${s}:"])}
+gpu:
+ hosts:
+ ${join("\n ", [for s in local.gpu_ips : "${s}:"])}
+
# using public IP of kube_server for ansible delegate_to
kube_server:
hosts:
@@ -302,6 +325,6 @@ resource "local_file" "hosts" {
${local.master_ip}
${local.ingress_ip}
${local.nfs_ip}
-${join("\n", local.worker_ips)}
+${join("\n", concat(local.worker_ips, local.gpu_ips))}
EOT
}
diff --git a/staging1/terraform/terraform.tfvars b/staging1/terraform/terraform.tfvars
index d1c9a0d..9ed9673 100644
--- a/staging1/terraform/terraform.tfvars
+++ b/staging1/terraform/terraform.tfvars
@@ -1,12 +1,12 @@
# These need to be defined for things to work
ip_pool = "PSNC-PUB1-EDU"
+net_name = ""
+net6_name = ""
site_name = "staging"
# These may need some adjustment for your provider
-master_cpus = 4
-master_ram = 8192
-worker_cpus = 4
-worker_ram = 8192
+master_flavor_name = "M1-NVME-2vCPU-8R-50D"
+worker_flavor_name = "M1-NVME-2vCPU-8R-50D"
# XXX: replace this for GPU flavor, once available
gpu_flavor_name = "M1-NVME-2vCPU-8R-50D"
diff --git a/staging1/terraform/vars.tf b/staging1/terraform/vars.tf
deleted file mode 100644
index 797ce3f..0000000
--- a/staging1/terraform/vars.tf
+++ /dev/null
@@ -1,64 +0,0 @@
-variable "ip_pool" {
- type = string
- description = "The name of the public IP pool for the servers"
-}
-
-variable "site_name" {
- type = string
- description = "Site identifier for internal host names"
-}
-
-variable "gpu_flavor_name" {
- type = string
- description = "Name of the GPU flavor"
-}
-
-variable "master_cpus" {
- type = number
- description = "Number of CPUs for the master"
-}
-
-variable "master_ram" {
- type = number
- description = "RAM for the master"
-}
-
-variable "worker_cpus" {
- type = number
- description = "Number of CPUs for the worker"
-}
-
-variable "worker_ram" {
- type = number
- description = "RAM for the worker"
-}
-
-variable "extra_workers" {
- type = number
- description = "Number of extra workers to create"
-}
-
-variable "gpu_workers" {
- type = number
- description = "Number of GPU workers to create"
-}
-
-variable "docker_volumes_size" {
- type = number
- description = "Size of volumes for docker (GB)"
-}
-
-variable "nfs_volume_size" {
- type = number
- description = "Size of volume for NFS server (GB)"
-}
-
-variable "scratch_volumes_size" {
- type = number
- description = "Size of volume for ephemeral volumes (GB)"
-}
-
-variable "squid_volume_size" {
- type = number
- description = "Size of volume for squid proxy, CVMFS cache (GB)"
-}
diff --git a/staging1/terraform/vars.tf b/staging1/terraform/vars.tf
new file mode 120000
index 0000000..00c4e3a
--- /dev/null
+++ b/staging1/terraform/vars.tf
@@ -0,0 +1 @@
+../../common/terraform/vars.tf
\ No newline at end of file
diff --git a/staging1/terraform/vms.tf b/staging1/terraform/vms.tf
index c3db9ec..c7fdcda 100644
--- a/staging1/terraform/vms.tf
+++ b/staging1/terraform/vms.tf
@@ -42,7 +42,7 @@ resource "openstack_networking_secgroup_rule_v2" "ping6" {
ethertype = "IPv6"
port_range_min = 128
port_range_max = 0
- protocol = "icmp"
+ protocol = "icmp" # icmp / ipv6-icmp
remote_ip_prefix = "::/0"
security_group_id = openstack_networking_secgroup_v2.ping.id
}
@@ -116,13 +116,11 @@ data "openstack_images_image_v2" "ubuntu" {
}
data "openstack_compute_flavor_v2" "master-flavor" {
- vcpus = var.master_cpus
- ram = var.master_ram
+ name = var.master_flavor_name
}
data "openstack_compute_flavor_v2" "worker-flavor" {
- vcpus = var.worker_cpus
- ram = var.worker_ram
+ name = var.worker_flavor_name
}
data "openstack_compute_flavor_v2" "gpu-flavor" {
diff --git a/staging2/terraform/terraform.tfvars b/staging2/terraform/terraform.tfvars
index 3462a54..41038d0 100644
--- a/staging2/terraform/terraform.tfvars
+++ b/staging2/terraform/terraform.tfvars
@@ -1,13 +1,12 @@
# These need to be defined for things to work
-ip_pool = "PSNC-PUB1-EDU"
+ip_pool = ""
net_name = "public"
+net6_name = ""
site_name = "staging2"
# These may need some adjustment for your provider
-master_cpus = 4
-master_ram = 8192
-worker_cpus = 4
-worker_ram = 8192
+master_flavor_name = "l2.c4r8.100"
+worker_flavor_name = "l2.c8r16.100"
# XXX: replace this for GPU flavor, once available
gpu_flavor_name = "l2.c2r4.100"
diff --git a/staging2/terraform/vars.tf b/staging2/terraform/vars.tf
deleted file mode 100644
index 797ce3f..0000000
--- a/staging2/terraform/vars.tf
+++ /dev/null
@@ -1,64 +0,0 @@
-variable "ip_pool" {
- type = string
- description = "The name of the public IP pool for the servers"
-}
-
-variable "site_name" {
- type = string
- description = "Site identifier for internal host names"
-}
-
-variable "gpu_flavor_name" {
- type = string
- description = "Name of the GPU flavor"
-}
-
-variable "master_cpus" {
- type = number
- description = "Number of CPUs for the master"
-}
-
-variable "master_ram" {
- type = number
- description = "RAM for the master"
-}
-
-variable "worker_cpus" {
- type = number
- description = "Number of CPUs for the worker"
-}
-
-variable "worker_ram" {
- type = number
- description = "RAM for the worker"
-}
-
-variable "extra_workers" {
- type = number
- description = "Number of extra workers to create"
-}
-
-variable "gpu_workers" {
- type = number
- description = "Number of GPU workers to create"
-}
-
-variable "docker_volumes_size" {
- type = number
- description = "Size of volumes for docker (GB)"
-}
-
-variable "nfs_volume_size" {
- type = number
- description = "Size of volume for NFS server (GB)"
-}
-
-variable "scratch_volumes_size" {
- type = number
- description = "Size of volume for ephemeral volumes (GB)"
-}
-
-variable "squid_volume_size" {
- type = number
- description = "Size of volume for squid proxy, CVMFS cache (GB)"
-}
diff --git a/staging2/terraform/vars.tf b/staging2/terraform/vars.tf
new file mode 120000
index 0000000..00c4e3a
--- /dev/null
+++ b/staging2/terraform/vars.tf
@@ -0,0 +1 @@
+../../common/terraform/vars.tf
\ No newline at end of file
diff --git a/staging2/terraform/vms.tf b/staging2/terraform/vms.tf
index bed0b9f..94976b0 100644
--- a/staging2/terraform/vms.tf
+++ b/staging2/terraform/vms.tf
@@ -107,22 +107,16 @@ resource "openstack_networking_secgroup_rule_v2" "https6" {
security_group_id = openstack_networking_secgroup_v2.http.id
}
-resource "openstack_networking_floatingip_v2" "public_ip" {
- pool = var.ip_pool
-}
-
data "openstack_images_image_v2" "ubuntu" {
name = "ubuntu-22.04"
}
data "openstack_compute_flavor_v2" "master-flavor" {
- vcpus = var.master_cpus
- ram = var.master_ram
+ name = var.master_flavor_name
}
data "openstack_compute_flavor_v2" "worker-flavor" {
- vcpus = var.worker_cpus
- ram = var.worker_ram
+ name = var.worker_flavor_name
}
data "openstack_compute_flavor_v2" "gpu-flavor" {
@@ -192,8 +186,9 @@ resource "openstack_compute_instance_v2" "gpu" {
}
resource "openstack_blockstorage_volume_v3" "nfs-volume" {
- name = "nfs"
- size = var.nfs_volume_size
+ name = "nfs"
+ size = var.nfs_volume_size
+ volume_type = "large"
}
resource "openstack_compute_volume_attach_v2" "nfs-volume-attach" {
@@ -256,8 +251,9 @@ EOT
}
resource "openstack_blockstorage_volume_v3" "squid-volume" {
- name = "squid"
- size = var.squid_volume_size
+ name = "squid"
+ size = var.squid_volume_size
+ volume_type = "fast"
}
resource "openstack_compute_volume_attach_v2" "squid-volume-attach" {
diff --git a/testing/terraform/terraform.tfvars b/testing/terraform/terraform.tfvars
index 6932d36..18fd4fa 100644
--- a/testing/terraform/terraform.tfvars
+++ b/testing/terraform/terraform.tfvars
@@ -5,10 +5,8 @@ net6_name = "public-muni-v6-432"
site_name = "testing"
# These may need some adjustment for your provider
-master_cpus = 2 # 2 CPUs to match existing flavours
-master_ram = 4096
-worker_cpus = 4
-worker_ram = 8192
+master_flavor_name = "standard.medium"
+worker_flavor_name = "standard.large"
gpu_flavor_name = "a3.32core-240ram-1t4"
# Number of extra workers
diff --git a/testing/terraform/vars.tf b/testing/terraform/vars.tf
deleted file mode 100644
index 4b29665..0000000
--- a/testing/terraform/vars.tf
+++ /dev/null
@@ -1,74 +0,0 @@
-variable "ip_pool" {
- type = string
- description = "The name of the public IP pool for the servers"
-}
-
-variable "net_name" {
- type = string
- description = "The name of the IPv4 network"
-}
-
-variable "net6_name" {
- type = string
- description = "The name of the IPv6 network"
-}
-
-variable "site_name" {
- type = string
- description = "Site identifier for internal host names"
-}
-
-variable "gpu_flavor_name" {
- type = string
- description = "Name of the GPU flavor"
-}
-
-variable "master_cpus" {
- type = number
- description = "Number of CPUs for the master"
-}
-
-variable "master_ram" {
- type = number
- description = "RAM for the master"
-}
-
-variable "worker_cpus" {
- type = number
- description = "Number of CPUs for the worker"
-}
-
-variable "worker_ram" {
- type = number
- description = "RAM for the worker"
-}
-
-variable "extra_workers" {
- type = number
- description = "Number of extra workers to create"
-}
-
-variable "gpu_workers" {
- type = number
- description = "Number of GPU workers to create"
-}
-
-variable "docker_volumes_size" {
- type = number
- description = "Size of volumes for docker (GB)"
-}
-
-variable "nfs_volume_size" {
- type = number
- description = "Size of volume for NFS server (GB)"
-}
-
-variable "scratch_volumes_size" {
- type = number
- description = "Size of volume for ephemeral volumes (GB)"
-}
-
-variable "squid_volume_size" {
- type = number
- description = "Size of volume for squid proxy, CVMFS cache (GB)"
-}
diff --git a/testing/terraform/vars.tf b/testing/terraform/vars.tf
new file mode 120000
index 0000000..00c4e3a
--- /dev/null
+++ b/testing/terraform/vars.tf
@@ -0,0 +1 @@
+../../common/terraform/vars.tf
\ No newline at end of file
diff --git a/testing/terraform/vms.tf b/testing/terraform/vms.tf
index 3628212..6003936 100644
--- a/testing/terraform/vms.tf
+++ b/testing/terraform/vms.tf
@@ -42,7 +42,7 @@ resource "openstack_networking_secgroup_rule_v2" "ping6" {
ethertype = "IPv6"
port_range_min = 128
port_range_max = 0
- protocol = "icmp"
+ protocol = "icmp" # icmp / ipv6-icmp
remote_ip_prefix = "::/0"
security_group_id = openstack_networking_secgroup_v2.ping.id
}
@@ -116,13 +116,11 @@ data "openstack_images_image_v2" "ubuntu" {
}
data "openstack_compute_flavor_v2" "master-flavor" {
- vcpus = var.master_cpus
- ram = var.master_ram
+ name = var.master_flavor_name
}
data "openstack_compute_flavor_v2" "worker-flavor" {
- vcpus = var.worker_cpus
- ram = var.worker_ram
+ name = var.worker_flavor_name
}
data "openstack_compute_flavor_v2" "gpu-flavor" {
@@ -197,7 +195,7 @@ resource "openstack_compute_instance_v2" "gpu" {
flavor_id = data.openstack_compute_flavor_v2.gpu-flavor.id
security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
user_data = file("cloud-init.yaml")
- tags = ["worker"]
+ tags = ["worker"]
network {
name = var.net_name
}
--
GitLab