diff --git a/ansible/roles/grafana/defaults/main.yml b/ansible/roles/grafana/defaults/main.yml
index 9e31211a9c..030d583346 100644
--- a/ansible/roles/grafana/defaults/main.yml
+++ b/ansible/roles/grafana/defaults/main.yml
@@ -83,13 +83,15 @@ grafana_default_volumes:
   - "{{ node_config_directory }}/grafana/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
-  - "grafana:/var/lib/grafana/"
   - "kolla_logs:/var/log/kolla/"
 grafana_extra_volumes: "{{ default_extra_volumes }}"
 
 grafana_start_first_node_delay: 10
 grafana_start_first_node_retries: 12
 
+# TODO(dawudm): make this True in the D release
+grafana_remove_old_volume: false
+
 ############
 # Prometheus
 ############
diff --git a/ansible/roles/grafana/tasks/post_config.yml b/ansible/roles/grafana/tasks/post_config.yml
index a59a689ceb..82993cecf7 100644
--- a/ansible/roles/grafana/tasks/post_config.yml
+++ b/ansible/roles/grafana/tasks/post_config.yml
@@ -13,6 +13,13 @@
   delay: 2
   run_once: true
 
+- name: Remove old grafana docker volume
+  become: true
+  kolla_container:
+    action: "remove_volume"
+    name: grafana
+  when: grafana_remove_old_volume | bool
+
 - name: Enable grafana datasources
   become: true
   kolla_toolbox:
diff --git a/releasenotes/notes/grafana-volume-81d569128d9e020f.yaml b/releasenotes/notes/grafana-volume-81d569128d9e020f.yaml
new file mode 100644
index 0000000000..f449fc34f9
--- /dev/null
+++ b/releasenotes/notes/grafana-volume-81d569128d9e020f.yaml
@@ -0,0 +1,23 @@
+---
+fixes:
+  - |
+    Fixes bug `#2039498
+    <https://bugs.launchpad.net/kolla-ansible/+bug/2039498>`__ where the
+    grafana docker volume was bind mounted over Grafana plugins installed at
+    image build time. This is fixed by copying the dashboards into the
+    container from an existing bind mount instead of using the ``grafana``
+    volume. This however leaves behind the volume which can be removed by
+    setting ``grafana_remove_old_volume`` to ``true``. Please note that any
+    plugins installed via the cli directly and not through kolla will be lost
+    when doing this. In a future release ``grafana_remove_old_volume`` will
+    default to ``true``.
+upgrade:
+  - |
+    The ``grafana`` volume is no longer used. If you wish to automatically
+    remove the old volume, set ``grafana_remove_old_volume`` to ``true``. Note
+    that doing this will lose any plugins installed via the cli directly and
+    not through kolla. If you have previously installed Grafana plugins via the
+    Grafana UI, or CLI, you must change to installing them at image `build time
+    <https://docs.openstack.org/kolla/latest/admin/image-building.html#dockerfile-customisation>`__.
+    The grafana volume, which will contain existing custom plugins, will be
+    automatically removed in the D release.