diff --git a/figures/network_topology.png b/figures/network_topology.png new file mode 100644 index 0000000000000000000000000000000000000000..9be438f5aa2011de40cbb52aa79bf9cd05fc63c4 Binary files /dev/null and b/figures/network_topology.png differ diff --git a/figures/network_topology_vm.png b/figures/network_topology_vm.png new file mode 100644 index 0000000000000000000000000000000000000000..edec41fb7e16dd9daf49b9c3556548e80e5e1bbc Binary files /dev/null and b/figures/network_topology_vm.png differ diff --git a/figures/scheme.png b/figures/scheme.png new file mode 100644 index 0000000000000000000000000000000000000000..006afb061f74732d52c986023db19952531aaea1 Binary files /dev/null and b/figures/scheme.png differ diff --git a/openstack_eosc.ipynb b/openstack_eosc.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..41e2050e193eb30d0617c8570e62d0175f7f162a --- /dev/null +++ b/openstack_eosc.ipynb @@ -0,0 +1,1088 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f1d44b71-9709-467e-92d4-c1faa31a93e7", + "metadata": { + "jp-MarkdownHeadingCollapsed": true + }, + "source": [ + "# Deploying Your First Virtual Machine on PSNC OpenStack\n", + "\n", + "Welcome to this hands-on tutorial for launching and configuring your first Virtual Machine (VM) in the PSNC OpenStack cloud environment. This guide will walk you through each step of the deployment processβfrom authentication to full VM provisioning and access.\n", + "\n", + "π **Further reading:** [PSNC OpenStack Virtual Machines](https://docs.psnc.pl/display/EOSCUserGuides/Virtual+Machines)\n" + ] + }, + { + "cell_type": "markdown", + "id": "00e621e9-5493-45aa-8cea-dcbb14ab0df6", + "metadata": {}, + "source": [ + "# Install the required libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "0ce0f97a-a87b-4655-8f85-cfabb07172f7", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: python-openstackclient in /opt/conda/lib/python3.12/site-packages (8.0.0)\n", + "Requirement already satisfied: python-keystoneclient in /opt/conda/lib/python3.12/site-packages (5.6.0)\n", + "Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.12/site-packages (8.1.5)\n", + "Requirement already satisfied: paramiko in /opt/conda/lib/python3.12/site-packages (3.5.1)\n", + "Requirement already satisfied: pbr!=2.1.0,>=2.0.0 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (6.1.1)\n", + "Requirement already satisfied: cryptography>=2.7 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (44.0.0)\n", + "Requirement already satisfied: cliff>=3.5.0 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (4.9.1)\n", + "Requirement already satisfied: iso8601>=0.1.11 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (2.1.0)\n", + "Requirement already satisfied: openstacksdk>=3.3.0 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (4.5.0)\n", + "Requirement already satisfied: osc-lib>=2.3.0 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (4.0.0)\n", + "Requirement already satisfied: oslo.i18n>=3.15.3 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (6.5.1)\n", + "Requirement already satisfied: python-cinderclient>=3.3.0 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (9.7.0)\n", + "Requirement already satisfied: requests>=2.27.0 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (2.32.3)\n", + "Requirement already satisfied: stevedore>=2.0.1 in /opt/conda/lib/python3.12/site-packages (from python-openstackclient) (5.4.1)\n", + "Requirement already satisfied: debtcollector>=1.2.0 in /opt/conda/lib/python3.12/site-packages (from python-keystoneclient) (3.0.0)\n", + "Requirement already satisfied: keystoneauth1>=3.4.0 in /opt/conda/lib/python3.12/site-packages (from python-keystoneclient) (5.10.0)\n", + "Requirement already satisfied: oslo.config>=5.2.0 in /opt/conda/lib/python3.12/site-packages (from python-keystoneclient) (9.7.1)\n", + "Requirement already satisfied: oslo.serialization>=2.18.0 in /opt/conda/lib/python3.12/site-packages (from python-keystoneclient) (5.7.0)\n", + "Requirement already satisfied: oslo.utils>=3.33.0 in /opt/conda/lib/python3.12/site-packages (from python-keystoneclient) (8.2.0)\n", + "Requirement already satisfied: packaging>=20.4 in /opt/conda/lib/python3.12/site-packages (from python-keystoneclient) (24.2)\n", + "Requirement already satisfied: comm>=0.1.3 in /opt/conda/lib/python3.12/site-packages (from ipywidgets) (0.2.2)\n", + "Requirement already satisfied: ipython>=6.1.0 in /opt/conda/lib/python3.12/site-packages (from ipywidgets) (8.31.0)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.12/site-packages (from ipywidgets) (5.14.3)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.12 in /opt/conda/lib/python3.12/site-packages (from ipywidgets) (4.0.13)\n", + "Requirement already satisfied: jupyterlab_widgets~=3.0.12 in /opt/conda/lib/python3.12/site-packages (from ipywidgets) (3.0.13)\n", + "Requirement already satisfied: bcrypt>=3.2 in /opt/conda/lib/python3.12/site-packages (from paramiko) (4.3.0)\n", + "Requirement already satisfied: pynacl>=1.5 in /opt/conda/lib/python3.12/site-packages (from paramiko) (1.5.0)\n", + "Requirement already satisfied: autopage>=0.4.0 in /opt/conda/lib/python3.12/site-packages (from cliff>=3.5.0->python-openstackclient) (0.5.2)\n", + "Requirement already satisfied: cmd2>=1.0.0 in /opt/conda/lib/python3.12/site-packages (from cliff>=3.5.0->python-openstackclient) (2.5.11)\n", + "Requirement already satisfied: PrettyTable>=0.7.2 in /opt/conda/lib/python3.12/site-packages (from cliff>=3.5.0->python-openstackclient) (3.16.0)\n", + "Requirement already satisfied: PyYAML>=3.12 in /opt/conda/lib/python3.12/site-packages (from cliff>=3.5.0->python-openstackclient) (6.0.2)\n", + "Requirement already satisfied: cffi>=1.12 in /opt/conda/lib/python3.12/site-packages (from cryptography>=2.7->python-openstackclient) (1.17.1)\n", + "Requirement already satisfied: wrapt>=1.7.0 in /opt/conda/lib/python3.12/site-packages (from debtcollector>=1.2.0->python-keystoneclient) (1.17.2)\n", + "Requirement already satisfied: decorator in /opt/conda/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.2)\n", + "Requirement already satisfied: matplotlib-inline in /opt/conda/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.7)\n", + "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", + "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /opt/conda/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.48)\n", + "Requirement already satisfied: pygments>=2.4.0 in /opt/conda/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (2.18.0)\n", + "Requirement already satisfied: stack_data in /opt/conda/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", + "Requirement already satisfied: os-service-types>=1.2.0 in /opt/conda/lib/python3.12/site-packages (from keystoneauth1>=3.4.0->python-keystoneclient) (1.7.0)\n", + "Requirement already satisfied: typing-extensions>=4.12 in /opt/conda/lib/python3.12/site-packages (from keystoneauth1>=3.4.0->python-keystoneclient) (4.12.2)\n", + "Requirement already satisfied: dogpile.cache>=0.6.5 in /opt/conda/lib/python3.12/site-packages (from openstacksdk>=3.3.0->python-openstackclient) (1.3.4)\n", + "Requirement already satisfied: jmespath>=0.9.0 in /opt/conda/lib/python3.12/site-packages (from openstacksdk>=3.3.0->python-openstackclient) (1.0.1)\n", + "Requirement already satisfied: jsonpatch!=1.20,>=1.16 in /opt/conda/lib/python3.12/site-packages (from openstacksdk>=3.3.0->python-openstackclient) (1.33)\n", + "Requirement already satisfied: platformdirs>=3 in /opt/conda/lib/python3.12/site-packages (from openstacksdk>=3.3.0->python-openstackclient) (4.3.6)\n", + "Requirement already satisfied: psutil>=3.2.2 in /opt/conda/lib/python3.12/site-packages (from openstacksdk>=3.3.0->python-openstackclient) (5.9.8)\n", + "Requirement already satisfied: requestsexceptions>=1.2.0 in /opt/conda/lib/python3.12/site-packages (from openstacksdk>=3.3.0->python-openstackclient) (1.4.0)\n", + "Requirement already satisfied: netaddr>=0.7.18 in /opt/conda/lib/python3.12/site-packages (from oslo.config>=5.2.0->python-keystoneclient) (1.3.0)\n", + "Requirement already satisfied: rfc3986>=1.2.0 in /opt/conda/lib/python3.12/site-packages (from oslo.config>=5.2.0->python-keystoneclient) (2.0.0)\n", + "Requirement already satisfied: msgpack>=0.5.2 in /opt/conda/lib/python3.12/site-packages (from oslo.serialization>=2.18.0->python-keystoneclient) (1.1.0)\n", + "Requirement already satisfied: tzdata>=2022.4 in /opt/conda/lib/python3.12/site-packages (from oslo.serialization>=2.18.0->python-keystoneclient) (2024.2)\n", + "Requirement already satisfied: pyparsing>=2.1.0 in /opt/conda/lib/python3.12/site-packages (from oslo.utils>=3.33.0->python-keystoneclient) (3.2.0)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.12/site-packages (from pbr!=2.1.0,>=2.0.0->python-openstackclient) (75.6.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.12/site-packages (from requests>=2.27.0->python-openstackclient) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.12/site-packages (from requests>=2.27.0->python-openstackclient) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.12/site-packages (from requests>=2.27.0->python-openstackclient) (2.3.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.12/site-packages (from requests>=2.27.0->python-openstackclient) (2024.12.14)\n", + "Requirement already satisfied: pycparser in /opt/conda/lib/python3.12/site-packages (from cffi>=1.12->cryptography>=2.7->python-openstackclient) (2.22)\n", + "Requirement already satisfied: pyperclip>=1.8 in /opt/conda/lib/python3.12/site-packages (from cmd2>=1.0.0->cliff>=3.5.0->python-openstackclient) (1.9.0)\n", + "Requirement already satisfied: wcwidth>=0.2.10 in /opt/conda/lib/python3.12/site-packages (from cmd2>=1.0.0->cliff>=3.5.0->python-openstackclient) (0.2.13)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /opt/conda/lib/python3.12/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.4)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.12/site-packages (from jsonpatch!=1.20,>=1.16->openstacksdk>=3.3.0->python-openstackclient) (3.0.0)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.12/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", + "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.12/site-packages (from stack_data->ipython>=6.1.0->ipywidgets) (2.1.0)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.12/site-packages (from stack_data->ipython>=6.1.0->ipywidgets) (3.0.0)\n", + "Requirement already satisfied: pure_eval in /opt/conda/lib/python3.12/site-packages (from stack_data->ipython>=6.1.0->ipywidgets) (0.2.3)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install python-openstackclient python-keystoneclient ipywidgets paramiko" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3718348-c7df-44d3-9811-66b7fa7ec7fb", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import paramiko\n", + "import yaml\n", + "from keystoneauth1.session import Session\n", + "from keystoneauth1.identity.v3.oidc import OidcAccessToken\n", + "from keystoneauth1.identity.v3.application_credential import ApplicationCredential\n", + "from keystoneclient.v3.client import Client\n", + "from openstack.connection import Connection\n", + "import ipywidgets as widgets\n", + "import base64\n", + "import time\n", + "from IPython.display import display, clear_output, Markdown\n", + "from openstack.exceptions import ResourceNotFound, SDKException" + ] + }, + { + "cell_type": "markdown", + "id": "02aedafa-9276-4350-b60c-53fe216514c9", + "metadata": {}, + "source": [ + "# π Connect to OpenStack\n", + "## Set up OpenStack session with OIDC token authentication\n", + "\n", + "To authenticate with OpenStack using OIDC (OpenID Connect), you'll need to provide the following parameters:\n", + "\n", + "- **`auth_url`**: The URL of the OpenStack authentication server. \n", + "\n", + "- **`protocol`**: The protocol used for authentication, such as `openid`.\n", + "\n", + "- **`identity_provider`**: The name of the trusted external Identity Provider (IdP) that authenticates your identity.\n", + "\n", + "- **`access_token`**: The token you receive from the IdP after successfully logging in. \n", + " This token proves your identity to OpenStack.\n", + "\n", + "> βοΈ These parameters are used to initialize an authenticated session with OpenStack through the `openstack.connect()` function.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5bc6fac0-c5fa-44a7-9d29-38ea5164e4bc", + "metadata": {}, + "outputs": [], + "source": [ + "def load_access_token(token_file_path=\"/var/run/secrets/oidc/access_token\"):\n", + " \"\"\"Reads access token from specified file\"\"\"\n", + " try:\n", + " return open(token_file_path, \"r\").read()\n", + " except IOError:\n", + " print(\"Reading from access token file failed.\", file=sys.stderr)\n", + "\n", + "cloud_creds = OidcAccessToken(auth_url=\"https://api.cloud.psnc.pl:5000/v3/\",\n", + " identity_provider=\"aai.open-science-cloud.ec.europa.eu\",\n", + " protocol=\"openid\",\n", + " access_token=load_access_token()\n", + " )\n", + "\n", + "cloud_session = Session(auth=cloud_creds)\n" + ] + }, + { + "cell_type": "markdown", + "id": "c27a6b45-cd62-446c-bccf-9cba425e54ab", + "metadata": {}, + "source": [ + "## OpenStack Keystone Client\n", + "\n", + "The Keystone client is used to interact with OpenStack's identity service and enables to:\n", + "\n", + "- Access OpenStack components not exposed in the primary API.\n", + "- Retrieve detailed information about your projects" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b871dc55-172a-43da-81d7-b7fa796a7c2c", + "metadata": {}, + "outputs": [], + "source": [ + "# Get list of accessible projects\n", + "keystone_client = Client(session=cloud_session)\n", + "my_projects = keystone_client.auth.projects()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "9a58d8c5-7dcd-4549-9b10-0456ccd66b30", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "## Scope to one of your projects you are entitled to access" + ], + "text/plain": [ + "<IPython.core.display.Markdown object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "90ef16930f794a71812097ee87883176", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Dropdown(description='Project:', layout=Layout(width='50%'), options=(('pp-0192f702-2833-4ab5-8c44-79f92eb323dβ¦" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "43118c111a1740f7898f72861426f2ca", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(button_style='success', description='Connect to project', style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1cd77110d36042578de07d7469959646", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "clear_output(wait=True) # Clear previous output first\n", + "display(Markdown(\"## Scope to one of your projects you are entitled to access\"))\n", + "\n", + "project_dropdown = widgets.Dropdown(\n", + " options=[(project.name, idx) for idx, project in enumerate(my_projects)],\n", + " description=\"Project:\",\n", + " layout=widgets.Layout(width='50%')\n", + ")\n", + "connect_button = widgets.Button(description=\"Connect to project\", button_style=\"success\")\n", + "output = widgets.Output()\n", + "\n", + "def on_connect_clicked(b):\n", + " with output:\n", + " clear_output()\n", + " idx = project_dropdown.value\n", + " selected_project = my_projects[idx]\n", + " print(f\"π Connecting to project: {selected_project.name} (ID: {selected_project.id}, Domain: {selected_project.domain_id})\")\n", + " scoped_cloud_creds = OidcAccessToken(\n", + " auth_url=\"https://api.cloud.psnc.pl:5000/v3/\",\n", + " identity_provider=\"aai.open-science-cloud.ec.europa.eu\",\n", + " protocol=\"openid\",\n", + " project_id=selected_project.id,\n", + " project_domain_id=selected_project.domain_id,\n", + " access_token=load_access_token()\n", + " )\n", + " global scoped_cloud_session, scoped_openstack_connection\n", + " scoped_cloud_session = Session(auth=scoped_cloud_creds)\n", + " scoped_openstack_connection = Connection(session=scoped_cloud_session)\n", + " print(\"β Connection established successfully!\")\n", + "\n", + "connect_button._click_handlers.callbacks.clear()\n", + "connect_button.on_click(on_connect_clicked)\n", + "display(project_dropdown, connect_button, output)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7c50e961-b797-4e1c-a91b-e44487324c0c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Your current project id: db6b4eb2d6fe454191e8d39b088564cd\n" + ] + } + ], + "source": [ + "print(\"Your current project id: \", scoped_openstack_connection.current_project_id) " + ] + }, + { + "cell_type": "markdown", + "id": "5a8c4205-e524-44ca-96fb-d7577bc74d90", + "metadata": {}, + "source": [ + "# Starting up your own VM\n", + "## ποΈ Create an SSH Key Pair for VM Access\n", + "\n", + "- If you already have a key, skip this step and provide the key name in the `create_server()` method.\n", + "- You can:\n", + " - Upload your **own** public key using `public_key=\"ssh-rsa ...\"` \n", + " - Or let **OpenStack generate one** for you (as weβll do below).\n", + "- Remember to save the private key securely! \n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "5e4b3a58-687b-471f-8c7f-a6db86bde820", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Private key saved to key.pem\n", + "Public key saved to key.pub\n" + ] + } + ], + "source": [ + "key_name = \"key\"\n", + "new_keypair = scoped_openstack_connection.create_keypair(key_name)\n", + "#Generating key-pair using existing public key\n", + "#scoped_openstack_connection.create_keypair(\"mykey2\", public_key=\"ssh-rsa ....\")\n", + "\n", + "#If you let OpenStack generate your key-pair you will need to save your\n", + "#private and public keys for later use\n", + "with open(f\"{key_name}.pem\", \"w\") as priv_file:\n", + " priv_file.write(new_keypair.private_key)\n", + " print(f\"Private key saved to {key_name}.pem\")\n", + "\n", + "with open(f\"{key_name}.pub\", \"w\") as pub_file:\n", + " pub_file.write(new_keypair.public_key)\n", + " print(f\"Public key saved to {key_name}.pub\")" + ] + }, + { + "cell_type": "markdown", + "id": "809ce42b-b70f-4dfb-82a9-cb505477663c", + "metadata": {}, + "source": [ + "## Create a Security Group for Your VM\n", + "\n", + "Before creating a virtual machine, it's important to define what kind of network traffic it should allow. \n", + "This is done using **Security Groups** in OpenStack.\n", + "\n", + "In this example, we:\n", + "\n", + "- Create a new security group\n", + "- Allow incoming **SSH** (port 22) traffic to enable remote access\n", + "- Allow incoming **ICMP** traffic so the VM can be pinged\n", + "- Allow incoming **HTTP** (port 80) traffic for hosting web services\n", + "\n", + "_Note: OpenStack automatically allows all outgoing traffic (egress) from your VM._\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "aceddf23-baeb-4e86-8100-52beec7241a4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "β Security group 'My VM Security Group' created and configured.\n" + ] + } + ], + "source": [ + "# Create a new security group\n", + "new_security_group = scoped_openstack_connection.create_security_group(\n", + " name=\"My VM Security Group\",\n", + " description=\"Allows SSH, HTTP and ICMP traffic\"\n", + ")\n", + "\n", + "# Allow incoming SSH (port 22)\n", + "scoped_openstack_connection.create_security_group_rule(\n", + " new_security_group.id,\n", + " port_range_min=22,\n", + " port_range_max=22,\n", + " protocol=\"tcp\",\n", + " direction=\"ingress\",\n", + " description=\"Allow SSH\"\n", + ")\n", + "# Allow incoming HTTP (port 80)\n", + "scoped_openstack_connection.network.create_security_group_rule(\n", + " security_group_id=new_security_group.id,\n", + " port_range_min=80,\n", + " port_range_max=80,\n", + " protocol=\"tcp\",\n", + " direction=\"ingress\",\n", + " description=\"Allow HTTP\"\n", + ")\n", + "# Allow incoming ICMP (ping)\n", + "scoped_openstack_connection.create_security_group_rule(\n", + " new_security_group.id,\n", + " protocol=\"icmp\",\n", + " direction=\"ingress\",\n", + " description=\"Allow ICMP\"\n", + ")\n", + "\n", + "print(f\"β Security group '{new_security_group.name}' created and configured.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "94a9ee51-c6d7-4c35-b969-5097bb777f1c", + "metadata": {}, + "source": [ + "## π Select Resources for Your VM\n", + "\n", + "Before launching a VM, you need to choose the following:\n", + "\n", + "- **Image (OS)**: The base operating system your VM will use.\n", + "- **Flavor (CPU/RAM/Disk)**: Determines how much compute power (vCPUs), memory (RAM), and disk your VM will have. \n", + " π [Flavor details at PSNC](https://docs.cloud.psnc.pl/en/general/regions/bst-pl2-region/)\n", + "\n", + "- **Network**: The virtual public network the VM will connect to, including internal/external IP options. \n", + " π [PSNC Network Design Overview](https://docs.psnc.pl/display/EOSCUserGuides/PSNC+Network+Design)\n", + "\n", + "> π οΈ Once selected, these options will be used to launch your VM using the OpenStack backend.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5e29662e-1576-48f8-9ae2-f9e2be0ffc60", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### Available Images" + ], + "text/plain": [ + "<IPython.core.display.Markdown object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "341b08877fcc44f7a71e223b2713cdf0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Dropdown(description='Image:', layout=Layout(width='85%'), options=('Windows 10 Pro 22H2 (2023-05-31) (ID: c91β¦" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "### Available Flavors" + ], + "text/plain": [ + "<IPython.core.display.Markdown object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a8ba85e215494b91a0b93a6e01128471", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Dropdown(description='Flavor:', layout=Layout(width='85%'), options=('C1-NET-16vCPU-32R (ID: c1-net-16-32)', 'β¦" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "### Available Networks" + ], + "text/plain": [ + "<IPython.core.display.Markdown object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3eefc30da3834df793223ad458b594a0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Dropdown(description='Network:', layout=Layout(width='85%'), options=('PSNC-STORAGE-MANILA (ID: 25cb17dc-9645-β¦" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def format_size_gb(size_bytes):\n", + " return f\"{size_bytes / (1024 ** 3):.2f} GB\"\n", + "\n", + "def format_image_options(images):\n", + " return [\n", + " f\"{image.name} (ID: {image.id}, Size: {format_size_gb(image.size)})\"\n", + " for image in images\n", + " ]\n", + "\n", + "def format_generic_options(resources, label_attr='name', id_attr='id'):\n", + " return [f\"{getattr(res, label_attr)} (ID: {getattr(res, id_attr)})\" for res in resources]\n", + " \n", + "images = list(scoped_openstack_connection.compute.images())\n", + "flavors = list(scoped_openstack_connection.compute.flavors())\n", + "networks = list(scoped_openstack_connection.network.networks())\n", + "\n", + "display(Markdown(\"### Available Images\"))\n", + "image_dropdown = widgets.Dropdown(\n", + " options=format_image_options(images),\n", + " description='Image:',\n", + " layout=widgets.Layout(width='85%')\n", + ")\n", + "display(image_dropdown)\n", + "\n", + "display(Markdown(\"### Available Flavors\"))\n", + "flavor_dropdown = widgets.Dropdown(\n", + " options=format_generic_options(flavors),\n", + " description='Flavor:',\n", + " layout=widgets.Layout(width='85%')\n", + ")\n", + "display(flavor_dropdown)\n", + "\n", + "display(Markdown(\"### Available Networks\"))\n", + "network_dropdown = widgets.Dropdown(\n", + " options=format_generic_options(networks),\n", + " description='Network:',\n", + " layout=widgets.Layout(width='85%')\n", + ")\n", + "display(network_dropdown)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c10e65f1-1e6e-44f4-8e86-aa6e5452d10c", + "metadata": {}, + "outputs": [], + "source": [ + "def get_selected_name(dropdown_value):\n", + " return dropdown_value.split(\" (ID: \")[0]\n", + "def get_selected_id(dropdown_value):\n", + " return dropdown_value.split(\" (ID: \")[1].split(\")\")[0]\n", + "selected_image = get_selected_name(image_dropdown.value)\n", + "selected_flavor = get_selected_name(flavor_dropdown.value)\n", + "selected_network = get_selected_name(network_dropdown.value)\n", + "selected_network_id = get_selected_id(network_dropdown.value)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "616210a6-a2f4-4732-bdbe-b37684d9dd74", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Selected configuration: PSNC-EXT-PUB1-EDU, M1-NVME-2vCPU-8R-50D and ubuntu-24.04-x86_64-server-cloudimg-20241105\n" + ] + } + ], + "source": [ + "print(f\"Selected configuration: {selected_network}, {selected_flavor} and {selected_image}\")" + ] + }, + { + "cell_type": "markdown", + "id": "92b4ca77-ecc9-44f0-8464-5548b604e518", + "metadata": {}, + "source": [ + "## π Set Up a Private Network Environment\n", + "You will also need to set up a **private network**, a **subnet**, and a **router** to manage traffic between your VMs and external networks. This process ensures that your VMs have both internal and external connectivity as required. \n", + "> Follow the steps below to configure your network:\n" + ] + }, + { + "cell_type": "markdown", + "id": "e91b0bbe-8561-4bbd-87f7-927e53c35f68", + "metadata": {}, + "source": [ + "<img src=\"figures/scheme.png\" alt=\"Network scheme\" width=\"400\" height=\"200\"/>\n" + ] + }, + { + "cell_type": "markdown", + "id": "c3b75537-fb1b-448d-ad03-d637abb68b11", + "metadata": {}, + "source": [ + "### Create Private Network" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "f9a37e64-ba85-43b3-b234-9ac09e743951", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating private network...\n", + "Created network: My private network (ID: fca7d4d2-4c2c-4a25-9ed8-9d65c9d0d52f)\n" + ] + } + ], + "source": [ + "try:\n", + " print(\"Creating private network...\")\n", + " network = scoped_openstack_connection.network.create_network(\n", + " name=\"My private network\",\n", + " admin_state_up=True,\n", + " port_security_enabled=True\n", + " )\n", + " print(f\"Created network: {network.name} (ID: {network.id})\")\n", + "except SDKException as e:\n", + " print(f\"Network creation failed: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "cb79e628-f76c-442c-b7f0-72e4d74a98ac", + "metadata": {}, + "source": [ + "### Create Internal Network (subnet)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "6a36a9da-14fc-41be-92a7-6e4b140c1d47", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating subnet...\n", + "Created subnet: My internal network (ID: 7558ff73-1885-4707-aa2e-bf9dbdd0fa19)\n" + ] + } + ], + "source": [ + "try:\n", + " print(\"Creating subnet...\")\n", + " subnet = scoped_openstack_connection.network.create_subnet(\n", + " network_id=network.id,\n", + " name=\"My internal network\",\n", + " ip_version=4,\n", + " cidr=\"192.168.1.0/24\",\n", + " )\n", + " print(f\"Created subnet: {subnet.name} (ID: {subnet.id})\")\n", + "except SDKException as e:\n", + " print(f\"Subnet creation failed: {e}\")\n", + " raise\n" + ] + }, + { + "cell_type": "markdown", + "id": "b6c347fc-c1a2-433e-9b71-4e20f93b9ac9", + "metadata": {}, + "source": [ + "### Create Router and connect with network" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "1720eb87-7fb2-420b-bac4-2b9d8f5c0d31", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating router with external network: PSNC-EXT-PUB1-EDU\n", + "Created router: My router (ID: 57abc611-b306-458e-bb17-cbdc49cdf9fb)\n", + "Connected subnet My internal network to router My router\n" + ] + } + ], + "source": [ + "try:\n", + " print(f\"Creating router with external network: {selected_network}\")\n", + " router = scoped_openstack_connection.network.create_router(\n", + " name=\"My router\",\n", + " external_gateway_info={\"network_id\": selected_network_id}\n", + " )\n", + " print(f\"Created router: {router.name} (ID: {router.id})\")\n", + "\n", + " # Connect private subnet to router\n", + " scoped_openstack_connection.network.add_interface_to_router(\n", + " router.id,\n", + " subnet_id=subnet.id\n", + " )\n", + " print(f\"Connected subnet {subnet.name} to router {router.name}\")\n", + "except SDKException as e:\n", + " print(f\"Router setup failed: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "3d1ab4c3-f811-459a-917b-4664f01253f7", + "metadata": {}, + "source": [ + "\n", + "> **Diagram**: OpenStack networking setup β private network connected with the external public network (e.g., `PSNC-EXT-PUB1-EDU`) by router.\n" + ] + }, + { + "cell_type": "markdown", + "id": "7483a3d0-7894-46f6-b202-128ff92b7276", + "metadata": {}, + "source": [ + "## π Launch VM" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "4cd699a0-ddd6-4356-b828-95673e15e405", + "metadata": {}, + "outputs": [], + "source": [ + "# VM parameters\n", + "new_volume = scoped_openstack_connection.create_volume(10)\n", + "vm_name = \"new_vm\"\n", + "server = scoped_openstack_connection.create_server(\n", + " name=vm_name,\n", + " image=selected_image,\n", + " flavor=selected_flavor,\n", + " network=network, # Use new private network\n", + " key_name=\"key\",\n", + " volumes=[new_volume],\n", + " security_groups=[new_security_group.name]\n", + ")\n", + "# Waiting until all is ready\n", + "time.sleep(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "47ac6e4e-ee8d-413b-a7bf-a7f1562d01e7", + "metadata": {}, + "outputs": [], + "source": [ + "# Refreshing VM object to get the current state\n", + "new_vm = scoped_openstack_connection.compute.find_server(server.id)" + ] + }, + { + "cell_type": "markdown", + "id": "eeae3963-20cf-4283-a40c-c6ab54ebea7d", + "metadata": {}, + "source": [ + "### Configure floating IP associated with the VM for an external access" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "b61224ce-446d-42f7-b3d6-12e204ef26b5", + "metadata": {}, + "outputs": [], + "source": [ + "new_fip = scoped_openstack_connection.create_floating_ip(network=selected_network)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "d547d657-15d5-4c9f-86f8-2ffd6f2f3466", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Floating IP: 62.3.174.55\n" + ] + } + ], + "source": [ + "floating_ip_address = new_fip.floating_ip_address\n", + "print(f\"Floating IP: {floating_ip_address}\")" + ] + }, + { + "cell_type": "markdown", + "id": "99447853-3a5e-445d-a690-8e034ef6156a", + "metadata": {}, + "source": [ + "> Next step is to add new FIP to new VM to get access through SSH." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "8b06239e-fae7-464e-a887-5ce9c36b124c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "openstack.compute.v2.server.Server(id=63542c19-0751-4b99-a42d-3d5cc22bb1a6, name=new_vm, status=BUILD, tenant_id=db6b4eb2d6fe454191e8d39b088564cd, user_id=0814e963545daece81ed1e1bda7b90ca33b58889e97e843aeb29ac634d15ae84, metadata={}, hostId=d54760d94ecc699c04cb77d71fa37050d12057e3460d7661477b5bf8, image={'id': '7816da3d-cc63-4f01-bc2a-832bf2391eb8', 'links': [{'rel': 'bookmark', 'href': 'https://claudius.cloud.psnc.pl:8774/images/7816da3d-cc63-4f01-bc2a-832bf2391eb8'}]}, flavor={'vcpus': 2, 'ram': 8192, 'disk': 50, 'ephemeral': 0, 'swap': 0, 'original_name': 'M1-NVME-2vCPU-8R-50D', 'extra_specs': {'aggregate_instance_extra_specs:nvme': 'general-purpose', 'quota:disk_read_bytes_sec': '524288000', 'quota:disk_read_iops_sec': '50000', 'quota:disk_write_bytes_sec': '524288000', 'quota:disk_write_iops_sec': '10000'}}, created=2025-04-14T08:26:48Z, updated=2025-04-14T08:26:52Z, addresses={}, accessIPv4=, accessIPv6=, links=[{'rel': 'self', 'href': 'https://claudius.cloud.psnc.pl:8774/v2.1/servers/63542c19-0751-4b99-a42d-3d5cc22bb1a6'}, {'rel': 'bookmark', 'href': 'https://claudius.cloud.psnc.pl:8774/servers/63542c19-0751-4b99-a42d-3d5cc22bb1a6'}], OS-DCF:diskConfig=MANUAL, progress=0, OS-EXT-AZ:availability_zone=BST0K10, pinned_availability_zone=BST0K10, config_drive=, key_name=key, OS-SRV-USG:launched_at=None, OS-SRV-USG:terminated_at=None, security_groups=[{'name': 'My VM Security Group'}], OS-EXT-STS:task_state=spawning, OS-EXT-STS:vm_state=building, OS-EXT-STS:power_state=0, os-extended-volumes:volumes_attached=[{'id': '6395a1f4-644e-4a6a-9d98-7892dba997b0', 'delete_on_termination': False}], locked=False, locked_reason=None, description=None, tags=[], trusted_image_certificates=None, OS-EXT-SRV-ATTR:hostname=new-vm, server_groups=[], location=Munch({'cloud': 'api.cloud.psnc.pl', 'region_name': None, 'zone': 'BST0K10', 'project': Munch({'id': 'db6b4eb2d6fe454191e8d39b088564cd', 'name': None, 'domain_id': None, 'domain_name': None})}))" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# add_ip_list requires actual address string\n", + "# or list of strings of address\n", + "scoped_openstack_connection.add_ip_list(new_vm, new_fip.floating_ip_address)" + ] + }, + { + "cell_type": "markdown", + "id": "9ac6fe06-da82-493b-a0ba-e6a4fd898a18", + "metadata": {}, + "source": [ + "\n", + "> **Diagram**: After successful launching of VM, our network topology is extended with new created instance\n" + ] + }, + { + "cell_type": "markdown", + "id": "ffd09a73-4a84-440e-8e8c-117b6b4163fc", + "metadata": {}, + "source": [ + "### π Connect to Your VM via SSH and configure web page\n", + "- Run a **cloud-init-like** script to:\n", + " - Update system packages\n", + " - Install **NGINX**\n", + " - Set up a simple homepage" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "43fbe01e-163a-44b1-ba1d-e7589562a918", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connecting to VM at 62.3.174.55...\n", + "Try accessing: http://62.3.174.55\n", + "Access VM via ssh: ssh -i mykey.pem ubuntu@62.3.174.55\n" + ] + } + ], + "source": [ + "print(f\"Connecting to VM at {floating_ip_address}...\")\n", + "ssh = paramiko.SSHClient()\n", + "ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n", + "try:\n", + " ssh.connect(\n", + " hostname=floating_ip_address,\n", + " username=\"ubuntu\",\n", + " key_filename=\"key.pem\" # \"/path/to/mykey.pem\"\n", + " )\n", + " cloud_init_script = \"\"\"\n", + " sudo apt-get update\n", + " sudo apt-get upgrade -y\n", + " sudo apt-get install -y nginx\n", + " sudo bash -c \"echo 'Hello from my VM' > /var/www/html/index.html\"\n", + " \"\"\"\n", + " ssh.exec_command(cloud_init_script)\n", + " ssh.close()\n", + "except Exception as e:\n", + " print(f\"SSH failed: {e}\")\n", + " raise\n", + "\n", + "print(f\"Try accessing: http://{floating_ip_address}\")\n", + "print(f\"Access VM via ssh: ssh -i mykey.pem ubuntu@{floating_ip_address}\")" + ] + }, + { + "cell_type": "markdown", + "id": "2e5e5e46-791d-406a-9f6f-be3d0c234b0d", + "metadata": {}, + "source": [ + "> It will take some time for script to finish, so the provided link will not be accessible instantly" + ] + }, + { + "cell_type": "markdown", + "id": "dd588470-3fc1-4b41-9bba-c929743fa9c9", + "metadata": {}, + "source": [ + "# Finish session" + ] + }, + { + "cell_type": "markdown", + "id": "89a921c0-8591-447a-b08a-ade26b7256ab", + "metadata": {}, + "source": [ + "## Detach floating IP from current server" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "3cf9a669-0dba-42aa-be26-e9f804c35e48", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scoped_openstack_connection.detach_ip_from_server(new_vm, new_fip.id)" + ] + }, + { + "cell_type": "markdown", + "id": "29e3db6c-5b15-4b32-ad7b-8356436733e7", + "metadata": {}, + "source": [ + "## Backup your attached volume" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "95feded0-22be-464c-876e-39f1015a773e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "openstack.block_storage.v3.backup.Backup(name=My Backup, volume_id=6395a1f4-644e-4a6a-9d98-7892dba997b0, description=None, force=True, is_incremental=False, snapshot_id=None, id=65fa853d-9dfd-4b67-85e8-c4e333a3107c, links=[{'rel': 'self', 'href': 'https://claudius.cloud.psnc.pl:8776/v3/db6b4eb2d6fe454191e8d39b088564cd/backups/65fa853d-9dfd-4b67-85e8-c4e333a3107c'}, {'rel': 'bookmark', 'href': 'https://claudius.cloud.psnc.pl:8776/db6b4eb2d6fe454191e8d39b088564cd/backups/65fa853d-9dfd-4b67-85e8-c4e333a3107c'}], status=available, size=10, object_count=205, availability_zone=BST, container=cinder-backup-s3-claudius, created_at=2025-04-14T08:45:35.000000, updated_at=2025-04-14T08:47:24.000000, fail_reason=None, has_dependent_backups=False, data_timestamp=2025-04-14T08:45:35.000000, metadata={}, location=Munch({'cloud': 'api.cloud.psnc.pl', 'region_name': None, 'zone': 'BST', 'project': Munch({'id': 'db6b4eb2d6fe454191e8d39b088564cd', 'name': None, 'domain_id': None, 'domain_name': None})}))" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "attached_volume = new_vm.attached_volumes[0]\n", + "# OpenStack recommends to suspend VM and then backup volume.\n", + "# You can use `force` parameter to bypass it.\n", + "scoped_openstack_connection.create_volume_backup(attached_volume.id, name=\"My Backup\", force=True)" + ] + }, + { + "cell_type": "markdown", + "id": "878db2a7-dbd3-41c8-b629-6ca2954381c6", + "metadata": {}, + "source": [ + "## Release allocated resources" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "301cbef1-ec92-439f-8f2d-fca92e2c33a9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scoped_openstack_connection.delete_server(new_vm.id)\n", + "scoped_openstack_connection.delete_floating_ip(new_fip)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "dc283305-7e0e-4967-84e9-d1573b45136e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# If you do not wish to use generated key-pair anymore\n", + "# you can simply delete it\n", + "scoped_openstack_connection.delete_keypair(\"mykey\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}