diff --git a/egi-devel/terraform/vms.tf b/common/terraform/vms-cesnet.tf similarity index 100% rename from egi-devel/terraform/vms.tf rename to common/terraform/vms-cesnet.tf diff --git a/egi-devel/terraform/vms-cesnet.tf b/egi-devel/terraform/vms-cesnet.tf new file mode 120000 index 0000000000000000000000000000000000000000..2e87c2727445749b5f5718e1f93e2214310ab3a0 --- /dev/null +++ b/egi-devel/terraform/vms-cesnet.tf @@ -0,0 +1 @@ +../../common/terraform/vms-cesnet.tf \ No newline at end of file diff --git a/egi-production/ansible.cfg b/egi-production/ansible.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c3a73bec9aa17dbdd27c77947d8813866f7036e6 --- /dev/null +++ b/egi-production/ansible.cfg @@ -0,0 +1,5 @@ +[defaults] +inventory=inventory + +[diff] +always=true diff --git a/egi-production/deploy.sh b/egi-production/deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..69903feaec518aa5f244e9ea3badb7d1dbcc9a97 --- /dev/null +++ b/egi-production/deploy.sh @@ -0,0 +1,58 @@ +#! /bin/bash -xe + +# +# Deploy EGI production instance +# + +cd terraform && terraform init && terraform apply +cd - +cp -pv terraform/inventory.yaml inventory/1-cesnet.yaml + +# dynamic DNS +ip="$(head -n 1 <terraform/fip.txt)" +shellstate=$(shopt -po xtrace) +set +o xtrace +# https://nsupdate.fedcloud.eu +vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/egi-production +FEDCLOUD_DYNAMIC_DNS=$(vault read -field data $vault_prefix/FEDCLOUD_DYNAMIC_DNS | grep ^map | head -n 1 | sed 's/map\[\(.*\)\]/\1/') +for auth in $FEDCLOUD_DYNAMIC_DNS; do + echo "curl -i -X GET -u $(echo "$auth" | cut -d: -f1):XXX https://nsupdate.fedcloud.eu/nic/update?myip=$ip" + curl -i -X GET -u "$auth" https://nsupdate.fedcloud.eu/nic/update?myip="$ip" +done +eval "$shellstate" +echo "Terraform finished. Check terraform/docker-volume.sh. Continue? (CTRL-C to quit)" +read -r _ + +# wait for ping and ssh +while read -r ip; do + while ! ping -c 1 "$ip"; do sleep 5; done + ssh-keygen -R "$ip" + while ! ssh egi@"$ip" -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no :; do sleep 10; done +done <terraform/fip.txt + +# check ssh access +ansible -m command -a 'uname -a' allnodes + +# wait cloud-init +ansible -m shell -a 'while ! test -f /var/lib/cloud/instance/boot-finished; do sleep 2; done' allnodes + +# setup volumes +ansible -m copy -a 'src=terraform/nfs-volume.sh dest=/root/ mode=preserve' nfs +ansible -m command -a '/root/nfs-volume.sh' nfs +ansible -m copy -a 'src=terraform/squid-volume.sh dest=/root/ mode=preserve' 'ingress[0]' +ansible -m command -a '/root/squid-volume.sh' 'ingress[0]' + +# kubernetes +ansible-playbook playbooks/k8s.yaml +while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -Ev ' (Running|Completed) '; do sleep 5; done +# docker runtime directory after Kubernetes deployment (problem with unmounts) +ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu' +ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu' +ansible-playbook playbooks/squid.yaml +ansible-playbook playbooks/cvmfs.yaml + +# image repository +ansible-playbook playbooks/repository-nexus.yaml + +# wait for finish +while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -Ev ' (Running|Completed) '; do sleep 5; done diff --git a/egi-production/inventory/1-cesnet.yaml b/egi-production/inventory/1-cesnet.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be90255f3d77d5cd3c895454b1150c1b6c2661c5 --- /dev/null +++ b/egi-production/inventory/1-cesnet.yaml @@ -0,0 +1,34 @@ +--- +fip: + hosts: + 78.128.235.186: + +master: + hosts: + 192.168.0.144: + # must be IPv4 address or hostname + kube_server: 192.168.0.144 + +ingress: + hosts: + 192.168.0.22: + +nfs: + hosts: + 192.168.0.42: + +worker: + hosts: + 192.168.0.122: + 192.168.0.137: + 192.168.0.181: + 192.168.0.38: + +gpu: + hosts: + +# using public IP of kube_server for ansible delegate_to +kube_server: + hosts: + 192.168.0.144: + ansible_host: 192.168.0.144 diff --git a/egi-production/inventory/99-all.yaml b/egi-production/inventory/99-all.yaml new file mode 100644 index 0000000000000000000000000000000000000000..501ec3cf2b6d35530d32676e3322cd771ed0ffd1 --- /dev/null +++ b/egi-production/inventory/99-all.yaml @@ -0,0 +1,28 @@ +--- +allnodes: + children: + master: + ingress: + nfs: + worker: + gpu: + +all: + vars: + ansible_become: true + ansible_user: egi + ansible_ssh_common_args: >- + -o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q egi@{{ groups["fip"][0] }}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + + site_name: egi-production + vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/egi-production + backup_repository: s3:s3.cl2.du.cesnet.cz/notebooks-production-g2 + + binder_hostname: binder-rc.egi.zcu.cz + notebooks_hostname: notebooks-rc.egi.zcu.cz + grafana_hostname: grafana.notebooks-rc.egi.zcu.cz + nexus_hostname: nexus.notebooks-rc.egi.zcu.cz + registry_binder_hostname: registry.binder-rc.egi.zcu.cz + registry_notebooks_hostname: registry.notebooks-rc.egi.zcu.cz diff --git a/egi-production/playbooks/accounting.yaml b/egi-production/playbooks/accounting.yaml new file mode 120000 index 0000000000000000000000000000000000000000..89284849af173ceeb1d8ceeece8f76cf7be45ea9 --- /dev/null +++ b/egi-production/playbooks/accounting.yaml @@ -0,0 +1 @@ +../../common/playbooks/accounting.yaml \ No newline at end of file diff --git a/egi-production/playbooks/backup.yaml b/egi-production/playbooks/backup.yaml new file mode 120000 index 0000000000000000000000000000000000000000..7456d1fcaf47db08c79c9595499014aad8980162 --- /dev/null +++ b/egi-production/playbooks/backup.yaml @@ -0,0 +1 @@ +../../common/playbooks/backup.yaml \ No newline at end of file diff --git a/egi-production/playbooks/cvmfs.yaml b/egi-production/playbooks/cvmfs.yaml new file mode 120000 index 0000000000000000000000000000000000000000..2e82cca6c387556209aad5ff66ea41eba8f28082 --- /dev/null +++ b/egi-production/playbooks/cvmfs.yaml @@ -0,0 +1 @@ +../../common/playbooks/cvmfs.yaml \ No newline at end of file diff --git a/egi-production/playbooks/files/calico.yaml b/egi-production/playbooks/files/calico.yaml new file mode 120000 index 0000000000000000000000000000000000000000..732c864b90be9b3f8aaefe227ad0da7a7685b763 --- /dev/null +++ b/egi-production/playbooks/files/calico.yaml @@ -0,0 +1 @@ +../../../common/playbooks/files/calico.yaml \ No newline at end of file diff --git a/egi-production/playbooks/files/egi-notebooks-privacy-policy.html b/egi-production/playbooks/files/egi-notebooks-privacy-policy.html new file mode 120000 index 0000000000000000000000000000000000000000..93c86977da70204e440a252a89c39b8fce6e97e8 --- /dev/null +++ b/egi-production/playbooks/files/egi-notebooks-privacy-policy.html @@ -0,0 +1 @@ +../../../common/playbooks/files/egi-notebooks-privacy-policy.html \ No newline at end of file diff --git a/egi-production/playbooks/files/egi-notebooks-terms-of-use.html b/egi-production/playbooks/files/egi-notebooks-terms-of-use.html new file mode 120000 index 0000000000000000000000000000000000000000..f8be3539b4213972ef86fee9785f256e93d0381f --- /dev/null +++ b/egi-production/playbooks/files/egi-notebooks-terms-of-use.html @@ -0,0 +1 @@ +../../../common/playbooks/files/egi-notebooks-terms-of-use.html \ No newline at end of file diff --git a/egi-production/playbooks/files/egi-style.css b/egi-production/playbooks/files/egi-style.css new file mode 120000 index 0000000000000000000000000000000000000000..b8f15e8edfa5a4514f144d9f5a44c2fe92153465 --- /dev/null +++ b/egi-production/playbooks/files/egi-style.css @@ -0,0 +1 @@ +../../../common/playbooks/files/egi-style.css \ No newline at end of file diff --git a/egi-production/playbooks/files/etc b/egi-production/playbooks/files/etc new file mode 120000 index 0000000000000000000000000000000000000000..ed53b8742792e16bb4bae2ed49d02c79d79de146 --- /dev/null +++ b/egi-production/playbooks/files/etc @@ -0,0 +1 @@ +../../../common/playbooks/files/etc \ No newline at end of file diff --git a/egi-production/playbooks/files/usr b/egi-production/playbooks/files/usr new file mode 120000 index 0000000000000000000000000000000000000000..b034223ec617fbae8cd736f29cf1c61394fa7ebd --- /dev/null +++ b/egi-production/playbooks/files/usr @@ -0,0 +1 @@ +../../../common/playbooks/files/usr \ No newline at end of file diff --git a/egi-production/playbooks/k8s.yaml b/egi-production/playbooks/k8s.yaml new file mode 120000 index 0000000000000000000000000000000000000000..117aed694c9e6d26908aec62da2dc03cdbbacc7d --- /dev/null +++ b/egi-production/playbooks/k8s.yaml @@ -0,0 +1 @@ +../../common/playbooks/k8s.yaml \ No newline at end of file diff --git a/egi-production/playbooks/notebooks-policies.yaml b/egi-production/playbooks/notebooks-policies.yaml new file mode 120000 index 0000000000000000000000000000000000000000..bd02904148d92a82b2caec03cde061a0d36b9b2c --- /dev/null +++ b/egi-production/playbooks/notebooks-policies.yaml @@ -0,0 +1 @@ +../../common/playbooks/notebooks-policies.yaml \ No newline at end of file diff --git a/egi-production/playbooks/notebooks.yaml b/egi-production/playbooks/notebooks.yaml new file mode 120000 index 0000000000000000000000000000000000000000..3f1a33f7bdd6b5a2381afcd25e41f8a051fde965 --- /dev/null +++ b/egi-production/playbooks/notebooks.yaml @@ -0,0 +1 @@ +../../common/playbooks/notebooks.yaml \ No newline at end of file diff --git a/egi-production/playbooks/public_keys b/egi-production/playbooks/public_keys new file mode 120000 index 0000000000000000000000000000000000000000..6ef4918a9eb6aba6c6076f8e4d42570f35735d86 --- /dev/null +++ b/egi-production/playbooks/public_keys @@ -0,0 +1 @@ +../../common/playbooks/public_keys \ No newline at end of file diff --git a/egi-production/playbooks/recover.yaml b/egi-production/playbooks/recover.yaml new file mode 120000 index 0000000000000000000000000000000000000000..cb9970b05b25ef0b6ad7f2956c8c6fdb068f31d2 --- /dev/null +++ b/egi-production/playbooks/recover.yaml @@ -0,0 +1 @@ +../../common/playbooks/recover.yaml \ No newline at end of file diff --git a/egi-production/playbooks/repository-nexus.yaml b/egi-production/playbooks/repository-nexus.yaml new file mode 120000 index 0000000000000000000000000000000000000000..e96cf70a09fd9d3246be0a44d89029743fa28bb8 --- /dev/null +++ b/egi-production/playbooks/repository-nexus.yaml @@ -0,0 +1 @@ +../../common/playbooks/repository-nexus.yaml \ No newline at end of file diff --git a/egi-production/playbooks/squid.yaml b/egi-production/playbooks/squid.yaml new file mode 120000 index 0000000000000000000000000000000000000000..114c327c31c35f2362f124c93f50c31d9e28b589 --- /dev/null +++ b/egi-production/playbooks/squid.yaml @@ -0,0 +1 @@ +../../common/playbooks/squid.yaml \ No newline at end of file diff --git a/egi-production/playbooks/subtasks b/egi-production/playbooks/subtasks new file mode 120000 index 0000000000000000000000000000000000000000..a6b9b950507954371fd7276e98270455915fcb4e --- /dev/null +++ b/egi-production/playbooks/subtasks @@ -0,0 +1 @@ +../../common/playbooks/subtasks \ No newline at end of file diff --git a/egi-production/playbooks/templates/backup.yaml b/egi-production/playbooks/templates/backup.yaml new file mode 120000 index 0000000000000000000000000000000000000000..df474ef121faa868b4d66b1f8f18d9df70746e0f --- /dev/null +++ b/egi-production/playbooks/templates/backup.yaml @@ -0,0 +1 @@ +../../../common/playbooks/templates/backup.yaml \ No newline at end of file diff --git a/egi-production/playbooks/templates/etc/exports b/egi-production/playbooks/templates/etc/exports new file mode 120000 index 0000000000000000000000000000000000000000..a743a02371266dc407172d7218cbce75da927810 --- /dev/null +++ b/egi-production/playbooks/templates/etc/exports @@ -0,0 +1 @@ +../../../../common/playbooks/templates/etc/exports.ipv4 \ No newline at end of file diff --git a/egi-production/playbooks/templates/etc/mailutils.conf b/egi-production/playbooks/templates/etc/mailutils.conf new file mode 120000 index 0000000000000000000000000000000000000000..dbd8a1f31599952201aa0ce602fc80b047c07c16 --- /dev/null +++ b/egi-production/playbooks/templates/etc/mailutils.conf @@ -0,0 +1 @@ +../../../../common/playbooks/templates/etc/mailutils.conf \ No newline at end of file diff --git a/egi-production/playbooks/templates/etc/squid b/egi-production/playbooks/templates/etc/squid new file mode 120000 index 0000000000000000000000000000000000000000..352b5984beddb28543e7e6e2a84e5d5152a155ec --- /dev/null +++ b/egi-production/playbooks/templates/etc/squid @@ -0,0 +1 @@ +../../../../common/playbooks/templates/etc/squid \ No newline at end of file diff --git a/egi-production/playbooks/templates/nexus b/egi-production/playbooks/templates/nexus new file mode 120000 index 0000000000000000000000000000000000000000..bc36c2ad67cee06927c92931f5ec258f22b4a72d --- /dev/null +++ b/egi-production/playbooks/templates/nexus @@ -0,0 +1 @@ +../../../common/playbooks/templates/nexus \ No newline at end of file diff --git a/egi-production/playbooks/templates/nexus.yaml b/egi-production/playbooks/templates/nexus.yaml new file mode 120000 index 0000000000000000000000000000000000000000..ff8f7a83f27197161310465ce54a672c9ab9c658 --- /dev/null +++ b/egi-production/playbooks/templates/nexus.yaml @@ -0,0 +1 @@ +../../../common/playbooks/templates/nexus.yaml \ No newline at end of file diff --git a/egi-production/playbooks/templates/recover.yaml b/egi-production/playbooks/templates/recover.yaml new file mode 120000 index 0000000000000000000000000000000000000000..646a5ae35717c5bce202bf8e9512f15f19e20fa2 --- /dev/null +++ b/egi-production/playbooks/templates/recover.yaml @@ -0,0 +1 @@ +../../../common/playbooks/templates/recover.yaml \ No newline at end of file diff --git a/egi-production/terraform/cloud-init.yaml b/egi-production/terraform/cloud-init.yaml new file mode 120000 index 0000000000000000000000000000000000000000..f315687671b2c747efb8a03d692c7bd8bd86bf91 --- /dev/null +++ b/egi-production/terraform/cloud-init.yaml @@ -0,0 +1 @@ +../../common/terraform/cloud-init.yaml \ No newline at end of file diff --git a/egi-production/terraform/firewall.tf b/egi-production/terraform/firewall.tf new file mode 120000 index 0000000000000000000000000000000000000000..0088c1251a5778555091eb81f3bf6840486531ef --- /dev/null +++ b/egi-production/terraform/firewall.tf @@ -0,0 +1 @@ +../../common/terraform/firewall.tf \ No newline at end of file diff --git a/egi-production/terraform/terraform.tfvars b/egi-production/terraform/terraform.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..162b4a23e31cb82e9557e657c7e0539c766bbbe0 --- /dev/null +++ b/egi-production/terraform/terraform.tfvars @@ -0,0 +1,34 @@ +# These need to be defined for things to work +ip_pool = "external-ipv4-general-public" +net_name = "group-project-network" +net6_name = "" +site_name = "egi-production" + +# These may need some adjustment for your provider +master_flavor_name = "e1.large" +worker_flavor_name = "c2.8core-30ram" +gpu_flavor_name = "a3.30core-240ram-nvidia-t4" + +# Number of extra workers +extra_workers = 4 + +# Number of GPU workers +gpu_workers = 0 + +# volumes for docker +docker_volumes_size = 384 + +# NFS volume +nfs_volume_size = 4096 + +# scratch volume +scratch_volumes_size = 0 + +# squid volume +squid_volume_size = 128 + +# global firewall rules - public and admin access +security_public_cidr = { + "0.0.0.0/0": "Public access", + "::/0": "Public access", +} diff --git a/egi-production/terraform/vars.tf b/egi-production/terraform/vars.tf new file mode 120000 index 0000000000000000000000000000000000000000..00c4e3a2893853ba74d00429d159cd321d0e7d78 --- /dev/null +++ b/egi-production/terraform/vars.tf @@ -0,0 +1 @@ +../../common/terraform/vars.tf \ No newline at end of file diff --git a/egi-production/terraform/versions.tf b/egi-production/terraform/versions.tf new file mode 120000 index 0000000000000000000000000000000000000000..b4eea0e6f9f99e56225b1308ca0813f3958b2474 --- /dev/null +++ b/egi-production/terraform/versions.tf @@ -0,0 +1 @@ +../../common/terraform/versions.tf \ No newline at end of file diff --git a/egi-production/terraform/vms-cesnet.tf b/egi-production/terraform/vms-cesnet.tf new file mode 120000 index 0000000000000000000000000000000000000000..2e87c2727445749b5f5718e1f93e2214310ab3a0 --- /dev/null +++ b/egi-production/terraform/vms-cesnet.tf @@ -0,0 +1 @@ +../../common/terraform/vms-cesnet.tf \ No newline at end of file