diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml
index ea9a4f265..26d6ba804 100644
--- a/zuul-tests.d/container-roles-jobs.yaml
+++ b/zuul-tests.d/container-roles-jobs.yaml
@@ -441,7 +441,8 @@
       This job tests changes to the ensure-kubernetes roles.  It
       is not meant to be used directly but rather run on changes to
       roles in the zuul-jobs repo.
-    tags: ubuntu-platforms
+    # Does not work under ubuntu-bionic
+    abstract: true
     files:
       - roles/ensure-docker/.*
       - roles/ensure-kubernetes/.*
@@ -450,21 +451,10 @@
     run: test-playbooks/ensure-kubernetes/crio.yaml
     post-run: test-playbooks/ensure-kubernetes/post.yaml
 
-- job:
-    name: zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic
-    description: Test the ensure-kubernetes role with crio-o on ubuntu-bionic
-    parent: zuul-jobs-test-ensure-kubernetes-crio
-    tags: auto-generated
-    nodeset:
-      nodes:
-        - name: ubuntu-bionic
-          label: ubuntu-bionic
-
 - job:
     name: zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal
     description: Test the ensure-kubernetes role with crio-o on ubuntu-focal
     parent: zuul-jobs-test-ensure-kubernetes-crio
-    tags: auto-generated
     nodeset:
       nodes:
         - name: ubuntu-focal
@@ -474,7 +464,6 @@
     name: zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy
     description: Test the ensure-kubernetes role with crio-o on ubuntu-jammy
     parent: zuul-jobs-test-ensure-kubernetes-crio
-    tags: auto-generated
     nodeset:
       nodes:
         - name: ubuntu-jammy
@@ -630,7 +619,6 @@
         - zuul-jobs-test-registry-buildset-registry-k8s-crio
         - zuul-jobs-test-registry-buildset-registry-openshift-docker
         - zuul-jobs-test-registry-buildset-registry-k8s-microk8s
-        - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy
         - zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy