diff --git a/test-playbooks/install-kubernetes/crio.yaml b/test-playbooks/install-kubernetes/crio.yaml
index f0ffc531e..7bf17420b 100644
--- a/test-playbooks/install-kubernetes/crio.yaml
+++ b/test-playbooks/install-kubernetes/crio.yaml
@@ -7,3 +7,28 @@
           - '1.1.1.1'
           - '8.8.8.8'
         kubernetes_runtime: cri-o
+  post_tasks:
+    - name: Check crio version
+      command: crictl version
+      become: yes
+      failed_when: false
+
+    - name: Create a test pod definition
+      copy:
+        dest: test-pod.yaml
+        content: |
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: test
+          spec:
+            restartPolicy: Never
+            containers:
+              - name: test
+                image: k8s.gcr.io/pause:3.1
+
+    - name: Start pod
+      command: kubectl apply -f test-pod.yaml
+
+    - name: Check status
+      shell: sleep 5; kubectl get pods
diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml
index a5f6ec06d..0ba34e7ec 100644
--- a/zuul-tests.d/container-roles-jobs.yaml
+++ b/zuul-tests.d/container-roles-jobs.yaml
@@ -36,7 +36,7 @@
     files:
       - roles/install-docker/.*
       - roles/install-kubernetes/.*
-      - test-playbooks/install-kubernetes.yaml
+      - test-playbooks/install-kubernetes/.*
     run: test-playbooks/install-kubernetes/docker.yaml
     post-run: test-playbooks/install-kubernetes/post.yaml
 
@@ -51,7 +51,7 @@
     files:
       - roles/install-docker/.*
       - roles/install-kubernetes/.*
-      - test-playbooks/install-kubernetes.yaml
+      - test-playbooks/install-kubernetes/.*
     run: test-playbooks/install-kubernetes/crio.yaml
     post-run: test-playbooks/install-kubernetes/post.yaml