diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ca17c8fb0bc3818b98e2f2e3cdfeb504f187a984..40d7b1926bafb0f7d293ceb656e5cf9f97878d17 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -43,6 +43,8 @@ test-single:
     # https://github.com/terraform-provider-openstack/terraform-provider-openstack/issues/1160
     - touch ./secure.yml
     - ip=`openstack floating ip create -c floating_ip_address -f value public-muni-147-251-21-GROUP`
+  after_script: &after
+    - openstack floating ip delete $ip
   cache:
     when: always
     key: single
@@ -56,7 +58,6 @@ test-single:
     - ssh-add ./ssh-key.terra1.txt
     - ssh -o PreferredAuthentications=publickey deployadm@$ip "sudo -u debian sh -c 'cd; ~deployadm/terraform/image/tests/run-tests.sh'"
     - terraform destroy $args
-    - openstack floating ip delete $ip
   rules:
     - if: '$JOB =~ /all|test-single/'
   artifacts: &artifacts
@@ -67,6 +68,7 @@ test-cluster:
   stage: test
   variables: *tfvars
   before_script: *before
+  after_script: *after
   cache:
     when: always
     key: cluster