diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 04f8b01313..fb157242cd 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -21,7 +21,7 @@ localhost       ansible_connection=local
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 [chrony-server:children]
-control
+haproxy
 
 [chrony:children]
 network
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 66ff970d6a..4cd55e27c2 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -41,7 +41,7 @@ monitoring
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 [chrony-server:children]
-control
+haproxy
 
 [chrony:children]
 control
diff --git a/ansible/roles/chrony/templates/chrony.conf.j2 b/ansible/roles/chrony/templates/chrony.conf.j2
index ece9a40190..592d65958e 100644
--- a/ansible/roles/chrony/templates/chrony.conf.j2
+++ b/ansible/roles/chrony/templates/chrony.conf.j2
@@ -1,13 +1,16 @@
-{% for host in groups['chrony-server'] %}
-{% if inventory_hostname != host %}
-server {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }} iburst
-{% endif %}
-{% endfor %}
+{% set keyfile = '/etc/chrony.keys' if kolla_base_distro in ['centos', 'oraclelinux', 'redhat'] else '/etc/chrony/chrony.keys' %}
+
+server {{ kolla_internal_vip_address }} iburst
+{# NOTE(jeffrey4l): external_ntp_servers may be None here #}
+{% if external_ntp_servers %}
 {% for ntp_server in external_ntp_servers %}
 server {{ ntp_server }} iburst
 {% endfor %}
+{% endif %}
 
-keyfile /etc/chrony/chrony.keys
+user chrony
+
+keyfile {{ keyfile }}
 
 commandkey 1
 
@@ -26,13 +29,16 @@ dumpdir /var/lib/chrony
 
 {% if inventory_hostname in groups['chrony-server'] %}
 allow all
+# prevent chrony sync from self
+deny {{ kolla_internal_vip_address }}
+deny {{ api_interface_address }}
 local stratum 10
 {% else %}
 port 0
 deny all
 {% endif %}
 
-bindaddress {{ api_interface_address }}
+bindaddress {{ kolla_internal_vip_address }}
 
 logchange 0.5
 
diff --git a/ansible/roles/chrony/templates/chrony.json.j2 b/ansible/roles/chrony/templates/chrony.json.j2
index 03f3ee9c7a..9322451f33 100644
--- a/ansible/roles/chrony/templates/chrony.json.j2
+++ b/ansible/roles/chrony/templates/chrony.json.j2
@@ -7,5 +7,17 @@
             "owner": "chrony",
             "perm": "0600"
         }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/chrony",
+            "owner": "chrony:kolla",
+            "recurse": true
+        },
+        {
+            "path": "/var/lib/chrony",
+            "owner": "chrony:chrony",
+            "recurse": true
+        }
     ]
 }
diff --git a/releasenotes/notes/move-chrony-server-group-depends-on-keepalived-27c60fbd1471cc29.yaml b/releasenotes/notes/move-chrony-server-group-depends-on-keepalived-27c60fbd1471cc29.yaml
new file mode 100644
index 0000000000..4edde34b17
--- /dev/null
+++ b/releasenotes/notes/move-chrony-server-group-depends-on-keepalived-27c60fbd1471cc29.yaml
@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    chrony server high available is implemented. And it depends on VIP now.
+    chrony-server group is moved to network node in default and must be the
+    same with haproxy group.