diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6d83b070..f2635b7b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,22 +32,12 @@ before_script:
     - apt-get update -q && apt-get -o dir::cache::archives="vendor/apt" install -yqq mysql-client
     - mysql --user=root --password="$MYSQL_ROOT_PASSWORD" --host="$DBHOST"
         -e "GRANT ALL ON test_$DBNAME.* TO '$DBUSER'@'%'"
+    # Remove the old test database if it has not been done yet
+    - mysql --user=root --password="$MYSQL_ROOT_PASSWORD" --host="$DBHOST"
+        -e "DROP DATABASE test_$DBNAME" || true
     - pip install --cache-dir vendor/pip -t vendor/python -r requirements-devel.txt
 
-stages:
-    - test
-    - cleanup
-
 test:
     stage: test
     script:
         - python manage.py test
-
-cleanup:
-    stage: cleanup
-    script:
-        # If an uncaught error occurs, the test database has to be destroyed
-        # manually
-        - mysql --user=root --password="$MYSQL_ROOT_PASSWORD" --host="$DBHOST"
-            -e "DROP DATABASE test_$DBNAME" || true
-    when: on_failure