Fix ironic inspector

Co-Authored-By: Jeffrey Zhang <zhang.lei.fly@gmail.com>
Depends-On: Id03619b4e26a0a77c2a39e2de21efd13be0e9200
Change-Id: I723345c1b23cb92ad94cefd965f07b94095ebb41
Closes-Bug: #1650345

ansible/group_vars/all.yml

@@ -188,6 +188,8 @@ murano_api_port: "8082"
 
 ironic_api_port: "6385"
 
+ironic_inspector_port: "5050"
+
 magnum_api_port: "9511"
 
 solum_application_deployment_port: "9777"

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -353,6 +353,11 @@ listen ironic_api
 {% for host in groups['ironic-api'] %}
   server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ ironic_api_port }} check inter 2000 rise 2 fall 5
 {% endfor %}
+listen ironic_inspector
+  bind {{ kolla_internal_vip_address }}:{{ ironic_inspector_port }}
+{% for host in groups['ironic-inspector'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ ironic_inspector_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
 {% if haproxy_enable_external_vip | bool %}
 
 listen ironic_api_external
@@ -360,6 +365,13 @@ listen ironic_api_external
 {% for host in groups['ironic-api'] %}
   server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ ironic_api_port }} check inter 2000 rise 2 fall 5
 {% endfor %}
+listen ironic_inspector_external
+  bind {{ kolla_external_vip_address }}:{{ ironic_inspector_port }} {{ tls_bind_info }}
+  http-request del-header X-Forwarded-Proto
+  http-request set-header X-Forwarded-Proto https if { ssl_fc }
+{% for host in groups['ironic-inspector'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ ironic_inspector_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
 {% endif %}
 {% endif %}
 

ansible/roles/ironic/defaults/main.yml

@@ -8,6 +8,10 @@ ironic_database_name: "ironic"
 ironic_database_user: "ironic"
 ironic_database_address: "{{ kolla_internal_fqdn }}:{{ database_port }}"
 
+ironic_inspector_database_name: "ironic_inspector"
+ironic_inspector_database_user: "ironic_inspector"
+ironic_inspector_database_address: "{{ kolla_internal_fqdn }}:{{ database_port }}"
+
 
 ####################
 # Docker
@@ -20,22 +24,43 @@ ironic_conductor_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{
 ironic_conductor_tag: "{{ openstack_release }}"
 ironic_conductor_image_full: "{{ ironic_conductor_image }}:{{ ironic_conductor_tag }}"
 
+ironic_pxe_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ironic-pxe"
+ironic_pxe_tag: "{{ openstack_release }}"
+ironic_pxe_image_full: "{{ ironic_pxe_image }}:{{ ironic_pxe_tag }}"
+
 ironic_inspector_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ironic-inspector"
 ironic_inspector_tag: "{{ openstack_release }}"
 ironic_inspector_image_full: "{{ ironic_inspector_image }}:{{ ironic_inspector_tag }}"
 
-ironic_pxe_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ironic-pxe"
+ironic_dnsmasq_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-dnsmasq"
-ironic_pxe_tag: "{{ openstack_release }}"
+ironic_dnsmasq_tag: "{{ openstack_release }}"
-ironic_pxe_image_full: "{{ ironic_pxe_image }}:{{ ironic_pxe_tag }}"
+ironic_dnsmasq_image_full: "{{ ironic_dnsmasq_image }}:{{ ironic_dnsmasq_tag }}"
 
 
 ####################
 # OpenStack
 ####################
+ironic_inspector_keystone_user: "ironic-inspector"
+
 ironic_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ ironic_api_port }}"
 ironic_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ ironic_api_port }}"
 ironic_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ ironic_api_port }}"
 
+ironic_inspector_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ ironic_inspector_port }}"
+ironic_inspector_internal_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ ironic_inspector_port }}"
+ironic_inspector_public_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ ironic_inspector_port }}"
+
 ironic_logging_debug: "{{ openstack_logging_debug }}"
 
 openstack_ironic_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}'}"
+
+openstack_ironic_inspector_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}'}"
+
+
+#########
+# Ironic
+#########
+
+ironic_dnsmasq_interface: "{{ api_interface }}"
+ironic_dnsmasq_dhcp_range:
+ironic_cleaning_network:

ansible/roles/ironic/tasks/bootstrap.yml

@@ -7,10 +7,15 @@
       login_port: "{{ database_port }}"
       login_user: "{{ database_user }}"
       login_password: "{{ database_password }}"
-      name: "{{ ironic_database_name }}"
+      name: "{{ item.database_name }}"
   register: database
   run_once: True
+  delegate_to: "{{ item.delegate_to }}"
+  with_items:
+    - database_name: "{{ ironic_database_name }}"
       delegate_to: "{{ groups['ironic-api'][0] }}"
+    - database_name: "{{ ironic_inspector_database_name }}"
+      delegate_to: "{{ groups['ironic-inspector'][0] }}"
 
 - name: Creating Ironic database user and setting permissions
   kolla_toolbox:
@@ -20,13 +25,20 @@
       login_port: "{{ database_port }}"
       login_user: "{{ database_user }}"
       login_password: "{{ database_password }}"
-      name: "{{ ironic_database_name }}"
+      name: "{{ item.database_name }}"
-      password: "{{ ironic_database_password }}"
+      password: "{{ item.database_password }}"
       host: "%"
-      priv: "{{ ironic_database_name }}.*:ALL"
+      priv: "{{ item.database_name }}.*:ALL"
       append_privs: "yes"
   run_once: True
+  delegate_to: "{{ item.delegate_to }}"
+  with_items:
+    - database_name: "{{ ironic_database_name }}"
+      database_password: "{{ ironic_database_password }}"
       delegate_to: "{{ groups['ironic-api'][0] }}"
+    - database_name: "{{ ironic_inspector_database_name }}"
+      database_password: "{{ ironic_inspector_database_password }}"
+      delegate_to: "{{ groups['ironic-inspector'][0] }}"
 
 - include: bootstrap_service.yml
   when: database.changed

ansible/roles/ironic/tasks/bootstrap_service.yml

@@ -17,3 +17,22 @@
       - "/etc/localtime:/etc/localtime:ro"
   run_once: True
   delegate_to: "{{ groups['ironic-api'][0] }}"
+
+- name: Running Ironic Inspector bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ ironic_inspector_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_ironic_inspector"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/ironic-inspector/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+  run_once: True
+  delegate_to: "{{ groups['ironic-inspector'][0] }}"

ansible/roles/ironic/tasks/config.yml

@@ -9,6 +9,7 @@
     - "ironic-conductor"
     - "ironic-inspector"
     - "ironic-pxe"
+    - "ironic-dnsmasq"
 
 - name: Copying over config.json files for services
   template:
@@ -19,6 +20,7 @@
     - "ironic-conductor"
     - "ironic-inspector"
     - "ironic-pxe"
+    - "ironic-dnsmasq"
 
 - name: Copying over ironic.conf
   merge_configs:
@@ -36,7 +38,46 @@
   with_items:
     - "ironic-api"
     - "ironic-conductor"
-    - "ironic-inspector"
+
+- name: Copying over inspector.conf
+  merge_configs:
+    vars:
+      service_name: "ironic-inspector"
+    sources:
+      - "{{ role_path }}/templates/ironic-inspector.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/database.conf"
+      - "{{ node_custom_config }}/messaging.conf"
+      - "{{ node_custom_config }}/ironic-inspector.conf"
+      - "{{ node_custom_config }}/ironic-inspector/inspector.conf"
+      - "{{ node_custom_config }}/ironic-inspector/{{ inventory_hostname }}/inspector.conf"
+    dest: "{{ node_config_directory }}/ironic-inspector/inspector.conf"
+
+- name: Copying over dnsmasq.conf
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/ironic-dnsmasq/dnsmasq.conf"
+  with_first_found:
+    - "{{ node_custom_config }}/ironic/ironic-dnsmasq.conf"
+    - "{{ node_custom_config }}/ironic/{{ inventory_hostname }}/ironic-dnsmasq.conf"
+    - "ironic-dnsmasq.conf.j2"
+
+- name: Copying pxelinux.cfg default
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/ironic-pxe/default"
+  with_first_found:
+    - "{{ node_custom_config }}/ironic/pxelinux.default"
+    - "{{ node_custom_config }}/ironic/{{ inventory_hostname }}/pxelinux.default"
+    - "pxelinux.default.j2"
+
+- name: Copying ironic-agent kernel and initramfs
+  copy:
+    src: "{{ node_custom_config }}/ironic/{{ item }}"
+    dest: "{{ node_config_directory }}/ironic-pxe/{{ item }}"
+  with_items:
+    - "ironic-agent.kernel"
+    - "ironic-agent.initramfs"
 
 - name: Check if policies shall be overwritten
   local_action: stat path="{{ node_custom_config }}/ironic/policy.json"

ansible/roles/ironic/tasks/deploy.yml

@@ -1,6 +1,7 @@
 ---
 - include: register.yml
-  when: inventory_hostname in groups['ironic-api']
+  when: inventory_hostname in groups['ironic-api'] or
+        inventory_hostname in groups['ironic-inspector']
 
 - include: config.yml
   when: inventory_hostname in groups['ironic-api'] or
@@ -9,7 +10,8 @@
         inventory_hostname in groups['ironic-pxe']
 
 - include: bootstrap.yml
-  when: inventory_hostname in groups['ironic-api']
+  when: inventory_hostname in groups['ironic-api'] or
+        inventory_hostname in groups['ironic-inspector']
 
 - include: start.yml
   when: inventory_hostname in groups['ironic-api'] or

ansible/roles/ironic/tasks/register.yml

@@ -17,6 +17,7 @@
   retries: 10
   delay: 5
   run_once: True
+  when: inventory_hostname in groups['ironic-api']
   with_items:
     - {'interface': 'admin', 'url': '{{ ironic_admin_endpoint }}'}
     - {'interface': 'internal', 'url': '{{ ironic_internal_endpoint }}'}
@@ -38,3 +39,46 @@
   retries: 10
   delay: 5
   run_once: True
+  when: inventory_hostname in groups['ironic-api']
+
+- name: Creating the Ironic Inspector service and endpoint
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_service
+    -a "service_name=ironic-inspector
+        service_type=baremetal-introspection
+        description='Ironic Inspector baremetal introspection service'
+        endpoint_region={{ openstack_region_name }}
+        url='{{ item.url }}'
+        interface='{{ item.interface }}'
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_ironic_inspector_auth }}' }}"
+    -e "{'openstack_ironic_inspector_auth':{{ openstack_ironic_inspector_auth }}}"
+  register: ironic_inspector_endpoint
+  changed_when: "{{ ironic_inspector_endpoint.stdout.find('localhost | SUCCESS => ') != -1 and (ironic_inspector_endpoint.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: ironic_inspector_endpoint.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True
+  when: inventory_hostname in groups['ironic-inspector']
+  with_items:
+    - {'interface': 'admin', 'url': '{{ ironic_inspector_admin_endpoint }}'}
+    - {'interface': 'internal', 'url': '{{ ironic_inspector_internal_endpoint }}'}
+    - {'interface': 'public', 'url': '{{ ironic_inspector_public_endpoint }}'}
+
+- name: Creating the Ironic Inspector project, user, and role
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_user
+    -a "project=service
+        user={{ ironic_inspector_keystone_user }}
+        password={{ ironic_inspector_keystone_password }}
+        role=admin
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_ironic_inspector_auth }}' }}"
+    -e "{'openstack_ironic_inspector_auth':{{ openstack_ironic_inspector_auth }}}"
+  register: ironic_inspector_user
+  changed_when: "{{ ironic_inspector_user.stdout.find('localhost | SUCCESS => ') != -1 and (ironic_inspector_user.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: ironic_inspector_user.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True
+  when: inventory_hostname in groups['ironic-inspector']

ansible/roles/ironic/tasks/start.yml

@@ -51,4 +51,16 @@
     volumes:
       - "{{ node_config_directory }}/ironic-inspector/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
   when: inventory_hostname in groups['ironic-inspector']
+
+- name: Staring ironic-dnsmasq container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ ironic_dnsmasq_image_full }}"
+    name: "ironic_dnsmasq"
+    volumes:
+      - "{{ node_config_directory }}/ironic-dnsmasq/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+  when: inventory_hostname in groups['ironic-conductor']

ansible/roles/ironic/templates/ironic-dnsmasq.conf.j2

@@ -0,0 +1,10 @@
+port=0
+interface={{ api_interface }}
+bind-interfaces
+dhcp-range={{ ironic_dnsmasq_dhcp_range }}
+dhcp-sequential-ip
+
+dhcp-option=option:tftp-server,{{ kolla_internal_vip_address }}
+dhcp-option=option:server-ip-address,{{ kolla_internal_vip_address }}
+dhcp-option=option:bootfile-name,pxelinux.0
+dhcp-option=210,/tftpboot/

ansible/roles/ironic/templates/ironic-dnsmasq.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "dnsmasq --no-daemon --conf-file=/etc/dnsmasq.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/dnsmasq.conf",
+            "dest": "/etc/dnsmasq.conf",
+            "owner": "root",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/ironic/templates/ironic-inspector.conf.j2

@@ -0,0 +1,35 @@
+[DEFAULT]
+debug = {{ ironic_logging_debug }}
+log_dir = /var/log/kolla/ironic
+
+listen_address = {{ api_interface_address }}
+listen_port = {{ ironic_inspector_port }}
+
+[ironic]
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+auth_type = password
+project_domain_id = default
+user_domain_id = default
+project_name = service
+username = {{ ironic_inspector_keystone_user }}
+password = {{ ironic_inspector_keystone_password }}
+
+[keystone_authtoken]
+auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+auth_type = password
+project_domain_id = default
+user_domain_id = default
+project_name = service
+username = {{ ironic_inspector_keystone_user }}
+password = {{ ironic_inspector_keystone_password }}
+
+memcache_security_strategy = ENCRYPT
+memcache_secret_key = {{ memcache_secret_key }}
+memcached_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
+
+[firewall]
+dnsmasq_interface = {{ ironic_dnsmasq_interface }}
+
+[database]
+connection = mysql+pymysql://{{ ironic_inspector_database_user }}:{{ ironic_inspector_database_password }}@{{ ironic_inspector_database_address }}/{{ ironic_inspector_database_name }}

ansible/roles/ironic/templates/ironic-inspector.json.j2

@@ -1,9 +1,9 @@
 {
-    "command": "ironic-inspector --config-file /etc/ironic-inspector/ironic.conf",
+    "command": "ironic-inspector --config-file /etc/ironic-inspector/inspector.conf",
     "config_files": [
         {
-            "source": "{{ container_config_directory }}/ironic.conf",
+            "source": "{{ container_config_directory }}/inspector.conf",
-            "dest": "/etc/ironic-inspector/ironic.conf",
+            "dest": "/etc/ironic-inspector/inspector.conf",
             "owner": "ironic",
             "perm": "0600"
         },

ansible/roles/ironic/templates/ironic-pxe.json.j2

@@ -1,4 +1,23 @@
 {
     "command": "/usr/sbin/in.tftpd --verbose --foreground --user root --address 0.0.0.0:69 --map-file /map-file /tftpboot",
-    "config_files": []
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/ironic-agent.kernel",
+            "dest": "/tftpboot/ironic-agent.kernel",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/ironic-agent.initramfs",
+            "dest": "/tftpboot/ironic-agent.initramfs",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/default",
+            "dest": "/tftpboot/pxelinux.cfg/default",
+            "owner": "root",
+            "perm": "0644"
+        }
+    ]
 }

ansible/roles/ironic/templates/ironic.conf.j2

@@ -21,18 +21,6 @@ api_url = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ ironic_api_port
 automated_clean=false
 {% endif %}
 
-{% if service_name == 'ironic-inspector' %}
-[ironic]
-os_auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}/v2.0
-os_username = {{ openstack_auth.username }}
-os_password = {{ openstack_auth.password }}
-os_tenant_name = {{ openstack_auth.project_name }}
-identity_uri = {{ openstack_auth.auth_url }}
-
-[firewall]
-dnsmasq_interface = {{ api_interface }}
-{% endif %}
-
 [database]
 connection = mysql+pymysql://{{ ironic_database_user }}:{{ ironic_database_password }}@{{ ironic_database_address }}/{{ ironic_database_name }}
 max_retries = -1
@@ -57,3 +45,7 @@ glance_host = {{ kolla_internal_fqdn }}
 
 [neutron]
 url = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ neutron_server_port }}
+cleaning_network = {{ ironic_cleaning_network }}
+
+[inspector]
+enabled = true

ansible/roles/ironic/templates/pxelinux.default.j2

@@ -0,0 +1,7 @@
+default introspect
+
+label introspect
+kernel ironic-agent.kernel
+append initrd=ironic-agent.initramfs ipa-inspection-callback-url=http://{{ kolla_internal_vip_address }}:{{ ironic_inspector_port }}/v1/continue systemd.journald.forward_to_console=yes
+
+ipappend 3

etc/kolla/passwords.yml

@@ -87,6 +87,9 @@ murano_keystone_password:
 ironic_database_password:
 ironic_keystone_password:
 
+ironic_inspector_database_password:
+ironic_inspector_keystone_password:
+
 magnum_database_password:
 magnum_keystone_password: