diff --git a/.gitignore b/.gitignore
index 15a6fcf101b4542345edf8fb1bad202aafdb8850..c35494c789829bc123264d2bcc0b167051931d2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,5 +3,6 @@ __pycache__/
.pytest_cache/
.vscode/
rwm.conf
+rwm-*.conf
testfile*
-venv/
\ No newline at end of file
+venv/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e0d9a0d6e6011dec31dffc57a9855f6e8db4bdef..8c2683c1e8fbfdf59be996713ff4fbbac4f3571e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,12 +4,15 @@ image: debian:bookworm
stages:
- code_quality
+variables:
+ GIT_STRATEGY: clone
+
code_quality:
stage: code_quality
before_script:
- - apt-get update && apt-get -y install make
- - make install
- - make venv
+ - python3 -m venv venv
+ - venv/bin/pip install -U pip
+ - venv/bin/pip install -r requirements.lock
script:
- . venv/bin/activate && make coverage
- . venv/bin/activate && make lint
diff --git a/Makefile b/Makefile
index cc7a31f5a8883e1fb310a06439bf17884d9e3912..fcee2a42cc1a6dcea744ae52def9b9db7a547581 100644
--- a/Makefile
+++ b/Makefile
@@ -1,21 +1,14 @@
-all: lint
+all: coverage lint
install:
- apt-get -y install awscli make python3-cryptography python3-tabulate rclone restic yamllint
+ apt-get -y install awscli python3-cryptography python3-tabulate rclone restic yamllint
-venv:
- apt-get -y install python3-venv
+install-dev:
+ apt-get -y install python3-venv snapd
python3 -m venv venv
venv/bin/pip install -U pip
venv/bin/pip install -r requirements.lock
-venv-refresh:
- apt-get -y install python3-venv
- rm -r venv
- python3 -m venv venv
- venv/bin/pip install -U pip
- venv/bin/pip install -r requirements.txt
-
freeze:
@pip freeze | grep -v '^pkg[-_]resources='
@@ -34,3 +27,24 @@ test:
coverage:
coverage run --source rwm -m pytest tests/ -x -vv
coverage report --show-missing --fail-under 100
+
+microceph-service:
+ snap install microceph
+ snap refresh --hold microceph
+ /snap/bin/microceph cluster bootstrap
+ /snap/bin/microceph disk add loop,1G,3
+ /snap/bin/microceph enable rgw
+ while true; do /snap/bin/ceph status | grep "HEALTH_OK" && break; done
+ # required for gitlab runner shell executor which runs as non-privileged user
+ ln -sf /var/snap/microceph/current/conf /etc/ceph
+ chmod 644 /etc/ceph/*
+
+microceph-cleanup:
+ snap remove microceph --purge
+ rm /etc/ceph
+
+microceph: microceph-cleanup microceph-service
+
+runner:
+ apt-get install -y ansible
+ ansible-playbook ansible/playbook_gitlab_runner.yml
diff --git a/README.md b/README.md
index 95314b40348705c8b154d15d4311ed6dab3f608d..5b0a1eb9f9d042a8efdcf9689cbd6fcb3da22248 100644
--- a/README.md
+++ b/README.md
@@ -113,14 +113,22 @@ rwm restic mount /mnt/restore
## Development
```
-git clone git@gitlab.flab.cesnet.cz:bodik/rwm.git /opt/rwm
+git clone git@gitlab.cesnet.cz:radoslav_bodo/rwm.git /opt/rwm
cd /opt/rwm
make install
-make venv
+make install-dev
+make microceph-service
. venv/bin/activate
+make coverage lint
```
-## Mainline backups
+## Gitlab Runner
-TBD
+```
+git clone git@gitlab.cesnet.cz:radoslav_bodo/rwm.git /opt/rwm
+cd /opt/rwm
+export RUNNER_URL=
+export RUNNER_TOKEN=
+make runner
+```
\ No newline at end of file
diff --git a/ansible/playbook_gitlab_runner.yml b/ansible/playbook_gitlab_runner.yml
new file mode 100644
index 0000000000000000000000000000000000000000..872beb9a713036892f40bc5c069970553284cffd
--- /dev/null
+++ b/ansible/playbook_gitlab_runner.yml
@@ -0,0 +1,63 @@
+---
+- name: install rwm runner
+ hosts: localhost
+ vars:
+ runner_url: "{{ lookup('ansible.builtin.env', 'RUNNER_URL') }}"
+ runner_token: "{{ lookup('ansible.builtin.env', 'RUNNER_TOKEN') }}"
+ runner_config: |
+ concurrent = 1
+ check_interval = 0
+ shutdown_timeout = 0
+ [session_server]
+ session_timeout = 1800
+ [[runners]]
+ name = "rwmsnaprunner"
+ url = "{{ runner_url }}"
+ token = "{{ runner_token }}"
+ executor = "shell"
+
+ handlers:
+ - name: gitlab-runner restart
+ service:
+ name: gitlab-runner
+ state: restarted
+
+ tasks:
+ - name: gitlab-runner dependencies
+ apt:
+ name:
+ - apt-transport-https
+ - software-properties-common
+ - wget
+ state: present
+
+ - name: gitlab-runner apt key
+ apt_key:
+ id: F6403F6544A38863DAA0B6E03F01618A51312F3F
+ url: https://packages.gitlab.com/runner/gitlab-runner/gpgkey
+ state: present
+
+ - name: gitlab-runner apt repo
+ apt_repository:
+ repo: "deb https://packages.gitlab.com/runner/gitlab-runner/debian bookworm main" # yamllint disable-line rule:line-length
+ state: present
+
+ - name: gitlab-runner package
+ apt:
+ name:
+ - gitlab-runner
+
+ - name: gitlab-runner config
+ copy:
+ content: "{{ runner_config }}"
+ dest: /etc/gitlab-runner/config.toml
+ owner: root
+ group: root
+ mode: 0600
+ notify: gitlab-runner restart
+
+ - name: development env with microceph
+ shell:
+ cmd: make install install-dev microceph-service
+ chdir: /opt/rwm
+ creates: /snap/bin/microceph
diff --git a/tests/conftest.py b/tests/conftest.py
index 770f0cfc175a9958793467024c7a7dd9f159a374..32e709475069970710e514a6bf3ab836d4f53153 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,14 +1,32 @@
"""pytest conftest"""
+import json
import os
import shutil
import socket
+import subprocess
from tempfile import mkdtemp
+import boto3
import pytest
from xprocess import ProcessStarter
+@pytest.fixture
+def tmpworkdir():
+ """
+ self cleaning temporary workdir
+ pytest tmpdir fixture has issues https://github.com/pytest-dev/pytest/issues/1120
+ """
+
+ cwd = os.getcwd()
+ tmpdir = mkdtemp(prefix='rwmtest_')
+ os.chdir(tmpdir)
+ yield tmpdir
+ os.chdir(cwd)
+ shutil.rmtree(tmpdir)
+
+
@pytest.fixture
def motoserver(xprocess):
"""mocking s3 server fixture"""
@@ -30,15 +48,48 @@ def motoserver(xprocess):
@pytest.fixture
-def tmpworkdir():
- """
- self cleaning temporary workdir
- pytest tmpdir fixture has issues https://github.com/pytest-dev/pytest/issues/1120
- """
+def microceph():
+ """microceph s3 server fixture"""
- cwd = os.getcwd()
- tmpdir = mkdtemp(prefix='rwmtest_')
- os.chdir(tmpdir)
- yield tmpdir
- os.chdir(cwd)
- shutil.rmtree(tmpdir)
+ yield "http://localhost:80"
+
+
+def rgwuser(microceph_url, name):
+ """rgwuser fixture"""
+
+ subprocess.run(
+ ["/snap/bin/radosgw-admin", "user", "rm", f"--uid={name}", "--purge-data"],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ check=False
+ )
+ proc = subprocess.run(
+ ["/snap/bin/radosgw-admin", "user", "create", f"--uid={name}", f"--display-name=rwguser_{name}"],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+ user = json.loads(proc.stdout)
+ yield boto3.resource(
+ 's3',
+ endpoint_url=microceph_url,
+ aws_access_key_id=user["keys"][0]["access_key"],
+ aws_secret_access_key=user["keys"][0]["secret_key"]
+ )
+
+ subprocess.run(["/snap/bin/radosgw-admin", "user", "rm", f"--uid={name}", "--purge-data"], check=True)
+
+
+@pytest.fixture
+def rgwuser_test1(microceph): # pylint: disable=redefined-outer-name, unused-argument
+ """rgwuser test1 stub"""
+
+ yield from rgwuser(microceph, "test1")
+
+
+@pytest.fixture
+def rgwuser_test2(microceph): # pylint: disable=redefined-outer-name, unused-argument
+ """rgwuser test2 stub"""
+
+ yield from rgwuser(microceph, "test2")
diff --git a/tests/test_policies.py b/tests/test_policies.py
new file mode 100644
index 0000000000000000000000000000000000000000..acd19db3f7d4ea38727ceca66287d8cefc5def2d
--- /dev/null
+++ b/tests/test_policies.py
@@ -0,0 +1,33 @@
+"""rwm bucket policies tests"""
+
+import boto3
+import pytest
+
+
+def test_microceph_defaults(
+ tmpworkdir: str,
+ microceph: str,
+ rgwuser_test1: boto3.resource,
+ rgwuser_test2: boto3.resource
+): # pylint: disable=unused-argument
+ """test microceph defaults"""
+
+ # bucket should not be present
+ test_bucket = "testbuckx"
+ assert test_bucket not in [x.name for x in rgwuser_test1.buckets.all()]
+
+ # create bucket
+ rgwuser_test1.create_bucket(Bucket=test_bucket)
+ assert test_bucket in [x.name for x in rgwuser_test1.buckets.all()]
+
+ # list from other identity, check it is not visible
+ assert test_bucket not in [x.name for x in rgwuser_test2.buckets.all()]
+ # but already exist
+ with pytest.raises(rgwuser_test2.meta.client.exceptions.BucketAlreadyExists):
+ rgwuser_test2.create_bucket(Bucket=test_bucket)
+
+ # belongs to expected user
+ assert rgwuser_test1.Bucket(test_bucket).Acl().owner["ID"] == "test1"
+ # but unaccessible by other user
+ with pytest.raises(rgwuser_test2.meta.client.exceptions.ClientError, match=r"AccessDenied"):
+ assert rgwuser_test2.Bucket(test_bucket).Acl().owner["ID"] == "test1"