diff --git a/playbooks/roles/bifrost-keystone-client-config/defaults/main.yml b/playbooks/roles/bifrost-keystone-client-config/defaults/main.yml
index 68be28aa6..d4528c0f4 100644
--- a/playbooks/roles/bifrost-keystone-client-config/defaults/main.yml
+++ b/playbooks/roles/bifrost-keystone-client-config/defaults/main.yml
@@ -5,6 +5,9 @@ ans_network_interface: "{{ network_interface | replace('-', '_') }}"
 internal_ip: "{{ hostvars[inventory_hostname]['ansible_' + ans_network_interface]['ipv4']['address'] }}"
 
 enable_tls: false
+enable_inspector: true
+enable_keystone: false
+skip_validation: false
 
 # Service URLs used for communication with them.
 api_protocol: "{{ 'https' if enable_tls | bool else 'http' }}"
diff --git a/playbooks/roles/bifrost-keystone-client-config/tasks/main.yml b/playbooks/roles/bifrost-keystone-client-config/tasks/main.yml
index 7f6bdc67f..e12f74b3e 100644
--- a/playbooks/roles/bifrost-keystone-client-config/tasks/main.yml
+++ b/playbooks/roles/bifrost-keystone-client-config/tasks/main.yml
@@ -63,3 +63,7 @@
     dest: "~{{ user | default('root') }}/openrc"
     owner: "{{ user | default('root') }}"
     mode: 0600
+
+- name: "Validate generated configuration"
+  import_tasks: validate.yml
+  when: not skip_validation | bool
diff --git a/playbooks/roles/bifrost-keystone-client-config/tasks/validate.yml b/playbooks/roles/bifrost-keystone-client-config/tasks/validate.yml
new file mode 100644
index 000000000..e811e2ac2
--- /dev/null
+++ b/playbooks/roles/bifrost-keystone-client-config/tasks/validate.yml
@@ -0,0 +1,60 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: "Determine the testing cloud name"
+  set_fact:
+    testing_cloud_name: "{{ (clouds | dict2items)[0].key }}"
+
+- block:
+    - name: "List bare metal nodes using CLI and clouds.yaml"
+      command: baremetal --debug node list
+      environment:
+        OS_CLOUD: "{{ testing_cloud_name }}"
+
+    - name: "List introspection rules using CLI and clouds.yaml"
+      command: baremetal --debug introspection rule list
+      environment:
+        OS_CLOUD: "{{ testing_cloud_name }}"
+      when: enable_inspector | bool
+
+    - name: "List services using CLI and clouds.yaml"
+      command: openstack --debug service list
+      environment:
+        OS_CLOUD: bifrost-admin
+      when:
+        - enable_keystone | bool
+        - "'bifrost-admin' in clouds"
+
+    - name: "List bare metal nodes using CLI and openrc"
+      shell: |
+        set -e
+        . ~/openrc {{ testing_cloud_name }}
+        baremetal --debug node list
+      args:
+        executable: /bin/bash
+
+    # FIXME(dtantsur): openrc provides no overrides for ironic-inspector, so we
+    # cannot test it.
+
+    - name: "List services using CLI and openrc"
+      shell: |
+        set -e
+        . ~/openrc bifrost-admin
+        openstack --debug service list
+      args:
+        executable: /bin/bash
+      when:
+        - enable_keystone | bool
+        - "'bifrost-admin' in clouds"
+  become: yes
+  become_user: "{{ user | default('root') }}"
diff --git a/playbooks/test-bifrost.yaml b/playbooks/test-bifrost.yaml
index 02ed9035e..5f7512e65 100644
--- a/playbooks/test-bifrost.yaml
+++ b/playbooks/test-bifrost.yaml
@@ -34,22 +34,6 @@
     - role: bifrost-test-inspection
       when: inspect_nodes | default(false) | bool
 
-- hosts: localhost
-  name: "Tests the use of openstack clients"
-  connection: local
-  tasks:
-    - name: "List bare metal nodes using openstack client"
-      command: baremetal --debug node list
-      environment:
-        OS_CLOUD: bifrost
-        PATH: /usr/local/bin:{{ ansible_env.PATH }}
-    - name: "List introspection rules using openstack client"
-      command: baremetal --debug introspection rule list
-      environment:
-        OS_CLOUD: bifrost
-        PATH: /usr/local/bin:{{ ansible_env.PATH }}
-      when: enable_inspector is defined and enable_inspector | bool
-
 - import_playbook: deploy-dynamic.yaml
 
 - hosts: baremetal
diff --git a/releasenotes/notes/cli-validate-474e15ba9631e72f.yaml b/releasenotes/notes/cli-validate-474e15ba9631e72f.yaml
new file mode 100644
index 000000000..2614a373e
--- /dev/null
+++ b/releasenotes/notes/cli-validate-474e15ba9631e72f.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    The ``bifrost-keystone-client-config`` role now validates that CLI
+    access actually works with the generated configuration, use
+    ``skip_validation=false`` to disable.