From 2f9cf0e3873b2671b5cc85af0bc443874c05364b Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Thu, 20 Aug 2020 13:22:41 -0500 Subject: [PATCH] Implement Nova - Add n-sch and n-meta-api - Add n-cond - Add novnc - Add n-api - Add n-cpu - Stop nova on host Change-Id: I6613bbe10502fa986c9f9961aae7e673ac462fc5 --- chart/templates/clusterrole.yaml | 1 + devstack/lib/neutron-legacy | 46 +++ devstack/lib/nova | 306 ++++++++++++------ devstack/plugin.sh | 13 +- images/nova/Dockerfile | 54 ++++ images/nova/bindep.txt | 15 + images/nova/build-requirements.txt | 1 + images/nova/constraints.txt | 1 + images/nova/nova-api-wsgi | 28 ++ images/nova/nova-compute | 29 ++ images/nova/nova-conductor | 29 ++ images/nova/nova-init | 32 ++ images/nova/nova-metadata-wsgi | 28 ++ images/nova/nova-novncproxy | 29 ++ images/nova/nova-scheduler | 29 ++ images/nova/requirements.txt | 8 + images/nova/setup-repos.sh | 91 ++++++ openstack_operator/barbican.py | 5 +- openstack_operator/cinder.py | 5 +- openstack_operator/database.py | 17 +- openstack_operator/glance.py | 5 +- openstack_operator/heat.py | 5 +- openstack_operator/keystone.py | 4 +- openstack_operator/magnum.py | 4 +- openstack_operator/neutron.py | 2 +- openstack_operator/nova.py | 60 +++- openstack_operator/objects.py | 4 + openstack_operator/operator.py | 4 +- openstack_operator/placement.py | 5 +- .../nova/compute-api/daemonset.yml.j2 | 84 +++++ .../templates/nova/compute-api/service.yml.j2 | 30 ++ .../templates/nova/compute/daemonset.yml.j2 | 102 ++++++ .../templates/nova/conductor/daemonset.yml.j2 | 68 ++++ .../templates/nova/conductor/job.yml.j2 | 51 +++ .../templates/nova/ingress.yml.j2 | 54 ++++ .../nova/metadata-api/daemonset.yml.j2 | 84 +++++ .../nova/metadata-api/service.yml.j2 | 30 ++ .../nova/novncproxy/daemonset.yml.j2 | 84 +++++ .../templates/nova/novncproxy/service.yml.j2 | 30 ++ .../templates/nova/scheduler/daemonset.yml.j2 | 68 ++++ playbooks/functional/devstack.yaml | 4 + zuul.d/functional-jobs.yaml | 15 +- zuul.d/neutron-jobs.yaml | 9 + zuul.d/nova-jobs.yaml | 60 ++++ 44 files changed, 1488 insertions(+), 145 deletions(-) create mode 100644 images/nova/Dockerfile create mode 100644 images/nova/bindep.txt create mode 100644 images/nova/build-requirements.txt create mode 100644 images/nova/constraints.txt create mode 100755 images/nova/nova-api-wsgi create mode 100755 images/nova/nova-compute create mode 100755 images/nova/nova-conductor create mode 100755 images/nova/nova-init create mode 100755 images/nova/nova-metadata-wsgi create mode 100755 images/nova/nova-novncproxy create mode 100755 images/nova/nova-scheduler create mode 100644 images/nova/requirements.txt create mode 100755 images/nova/setup-repos.sh create mode 100644 openstack_operator/templates/nova/compute-api/daemonset.yml.j2 create mode 100644 openstack_operator/templates/nova/compute-api/service.yml.j2 create mode 100644 openstack_operator/templates/nova/compute/daemonset.yml.j2 create mode 100644 openstack_operator/templates/nova/conductor/daemonset.yml.j2 create mode 100644 openstack_operator/templates/nova/conductor/job.yml.j2 create mode 100644 openstack_operator/templates/nova/ingress.yml.j2 create mode 100644 openstack_operator/templates/nova/metadata-api/daemonset.yml.j2 create mode 100644 openstack_operator/templates/nova/metadata-api/service.yml.j2 create mode 100644 openstack_operator/templates/nova/novncproxy/daemonset.yml.j2 create mode 100644 openstack_operator/templates/nova/novncproxy/service.yml.j2 create mode 100644 openstack_operator/templates/nova/scheduler/daemonset.yml.j2 create mode 100644 zuul.d/nova-jobs.yaml diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml index 76706cbf..62dd89fa 100644 --- a/chart/templates/clusterrole.yaml +++ b/chart/templates/clusterrole.yaml @@ -76,6 +76,7 @@ rules: - batch resources: - cronjobs + - jobs verbs: - create - delete diff --git a/devstack/lib/neutron-legacy b/devstack/lib/neutron-legacy index 70ac2645..1abd9a85 100644 --- a/devstack/lib/neutron-legacy +++ b/devstack/lib/neutron-legacy @@ -251,3 +251,49 @@ function _neutron_ovs_base_configure_l3_agent { echo noop } export -f _neutron_ovs_base_configure_l3_agent + +function _create_nova_configs_neutron { + + local conf=${1:-$NOVA_CONF} + kubernetes_ensure_resource secret/neutron-application-credential + NEUTRON_APPLICATION_CREDENTIAL_SECRET=$(get_data_from_secret neutron-application-credential openstack secret) + NEUTRON_APPLICATION_CREDENTIAL_ID=$(get_data_from_secret neutron-application-credential openstack id) + + iniset $conf neutron auth_url $KEYSTONE_AUTH_URI_V3 + iniset $conf neutron auth_type v3applicationcredential + iniset $conf neutron application_credential_id $NEUTRON_APPLICATION_CREDENTIAL_ID + iniset $conf neutron application_credential_secret $NEUTRON_APPLICATION_CREDENTIAL_SECRET + iniset $conf neutron auth_strategy "$Q_AUTH_STRATEGY" + + # optionally set options in nova_conf + neutron_plugin_create_nova_conf $conf + + if is_service_enabled q-meta; then + iniset $conf neutron service_metadata_proxy "True" + fi + + iniset $conf DEFAULT vif_plugging_is_fatal "$VIF_PLUGGING_IS_FATAL" + iniset $conf DEFAULT vif_plugging_timeout "$VIF_PLUGGING_TIMEOUT" +} + +function create_nova_conf_neutron { + _create_nova_configs_neutron $NOVA_CONF + _create_nova_configs_neutron $NOVA_CPU_CONF +} +export -f create_nova_conf_neutron + + +# NOTE(Alex): Change the nova metadata host +function _configure_neutron_metadata_agent { + cp $NEUTRON_DIR/etc/metadata_agent.ini.sample $Q_META_CONF_FILE + + iniset $Q_META_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL + iniset $Q_META_CONF_FILE DEFAULT nova_metadata_host nova-metadata.openstack.svc + iniset $Q_META_CONF_FILE DEFAULT nova_metadata_port 80 + iniset $Q_META_CONF_FILE DEFAULT metadata_workers $API_WORKERS + iniset $Q_META_CONF_FILE AGENT root_helper "$Q_RR_COMMAND" + if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then + iniset $Q_META_CONF_FILE AGENT root_helper_daemon "$Q_RR_DAEMON_COMMAND" + fi +} +export -f _configure_neutron_metadata_agent diff --git a/devstack/lib/nova b/devstack/lib/nova index 5963e526..68d5e0cf 100644 --- a/devstack/lib/nova +++ b/devstack/lib/nova @@ -14,6 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. +NOVA_STATE_PATH=/var/lib/nova +# INSTANCES_PATH is the previous name for this +NOVA_INSTANCES_PATH=$NOVA_STATE_PATH/instances + function create_nova_conf { # Remove legacy ``nova.conf`` rm -f $NOVA_DIR/bin/nova.conf @@ -25,7 +29,6 @@ function create_nova_conf { iniset $NOVA_CONF DEFAULT allow_resize_to_same_host "True" fi iniset $NOVA_CONF wsgi api_paste_config "$NOVA_API_PASTE_INI" - iniset $NOVA_CONF DEFAULT rootwrap_config "$NOVA_CONF_DIR/rootwrap.conf" iniset $NOVA_CONF filter_scheduler enabled_filters "$NOVA_FILTERS" iniset $NOVA_CONF scheduler workers "$API_WORKERS" iniset $NOVA_CONF neutron default_floating_pool "$PUBLIC_NETWORK_NAME" @@ -47,56 +50,29 @@ function create_nova_conf { iniset $NOVA_CONF DEFAULT bindir "/usr/bin" fi - # only setup database connections and cache backend if there are services - # that require them running on the host. The ensures that n-cpu doesn't - # leak a need to use the db in a multinode scenario. - if is_service_enabled n-api n-cond n-sched; then - # If we're in multi-tier cells mode, we want our control services pointing - # at cell0 instead of cell1 to ensure isolation. If not, we point everything - # at the main database like normal. - if [[ "$CELLSV2_SETUP" == "singleconductor" ]]; then - local db="nova_cell1" - else - local db="nova_cell0" - # When in superconductor mode, nova-compute can't send instance - # info updates to the scheduler, so just disable it. - iniset $NOVA_CONF filter_scheduler track_instance_changes False - fi + kubernetes_ensure_resource secret/nova-cell1-mysql + NOVA_CELL1_DATABASE_USER=$(get_data_from_secret nova-cell1-mysql openstack USER) + NOVA_CELL1_DATABASE_PASSWORD=$(get_data_from_secret nova-cell1-mysql openstack PASSWORD) + NOVA_CELL1_DATABASE_NAME=$(get_data_from_secret nova-cell1-mysql openstack DATABASE) + iniset $NOVA_CONF database connection "mysql+pymysql://$NOVA_CELL1_DATABASE_USER:$NOVA_CELL1_DATABASE_PASSWORD@nova-cell1-mysql-master.openstack.svc/$NOVA_CELL1_DATABASE_NAME?charset=utf8" - kubernetes_ensure_resource secret/nova-cell1-mysql - NOVA_CELL1_DATABASE_USER=$(get_data_from_secret nova-cell1-mysql openstack USER) - NOVA_CELL1_DATABASE_PASSWORD=$(get_data_from_secret nova-cell1-mysql openstack PASSWORD) - NOVA_CELL1_DATABASE_NAME=$(get_data_from_secret nova-cell1-mysql openstack DATABASE) - iniset $NOVA_CONF database connection "mysql+pymysql://$NOVA_CELL1_DATABASE_USER:$NOVA_CELL1_DATABASE_PASSWORD@nova-cell1-mysql-master.openstack.svc/$NOVA_CELL1_DATABASE_NAME?charset=utf8" + kubernetes_ensure_resource secret/nova-api-mysql + NOVA_API_DATABASE_USER=$(get_data_from_secret nova-api-mysql openstack USER) + NOVA_API_DATABASE_PASSWORD=$(get_data_from_secret nova-api-mysql openstack PASSWORD) + NOVA_API_DATABASE_NAME=$(get_data_from_secret nova-api-mysql openstack DATABASE) + iniset $NOVA_CONF api_database connection "mysql+pymysql://$NOVA_API_DATABASE_USER:$NOVA_API_DATABASE_PASSWORD@nova-api-mysql-master.openstack.svc/$NOVA_API_DATABASE_NAME?charset=utf8" - kubernetes_ensure_resource secret/nova-api-mysql - NOVA_API_DATABASE_USER=$(get_data_from_secret nova-api-mysql openstack USER) - NOVA_API_DATABASE_PASSWORD=$(get_data_from_secret nova-api-mysql openstack PASSWORD) - NOVA_API_DATABASE_NAME=$(get_data_from_secret nova-api-mysql openstack DATABASE) - iniset $NOVA_CONF api_database connection "mysql+pymysql://$NOVA_API_DATABASE_USER:$NOVA_API_DATABASE_PASSWORD@nova-api-mysql-master.openstack.svc/$NOVA_API_DATABASE_NAME?charset=utf8" + # Cache related settings + # Those settings aren't really needed in n-cpu thus it is configured + # only on nodes which runs controller services + iniset $NOVA_CONF cache enabled $NOVA_ENABLE_CACHE + iniset $NOVA_CONF cache backend $CACHE_BACKEND + iniset $NOVA_CONF cache memcache_servers mcrouter-memcached-nova.openstack.svc.cluster.local - # Cache related settings - # Those settings aren't really needed in n-cpu thus it is configured - # only on nodes which runs controller services - iniset $NOVA_CONF cache enabled $NOVA_ENABLE_CACHE - iniset $NOVA_CONF cache backend $CACHE_BACKEND - iniset $NOVA_CONF cache memcache_servers mcrouter-memcached-nova.openstack.svc.cluster.local - fi + NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/,metadata//") + iniset $NOVA_CONF DEFAULT enabled_apis "$NOVA_ENABLED_APIS" - if is_service_enabled n-api; then - if is_service_enabled n-api-meta; then - # If running n-api-meta as a separate service - NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/,metadata//") - fi - iniset $NOVA_CONF DEFAULT enabled_apis "$NOVA_ENABLED_APIS" - if is_service_enabled tls-proxy && [ "$NOVA_USE_MOD_WSGI" == "False" ]; then - # Set the service port for a proxy to take the original - iniset $NOVA_CONF DEFAULT osapi_compute_listen_port "$NOVA_SERVICE_PORT_INT" - iniset $NOVA_CONF DEFAULT osapi_compute_link_prefix $NOVA_SERVICE_PROTOCOL://$NOVA_SERVICE_HOST:$NOVA_SERVICE_PORT - fi - - configure_keystone_authtoken_middleware $NOVA_CONF nova - fi + configure_keystone_authtoken_middleware $NOVA_CONF nova if is_service_enabled cinder; then configure_cinder_access @@ -121,12 +97,9 @@ function create_nova_conf { iniset $NOVA_CONF DEFAULT mkisofs_cmd /usr/bin/mkisofs fi - # Format logging - setup_logging $NOVA_CONF - iniset $NOVA_CONF upgrade_levels compute "auto" - write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute" + proxy_pass_to_kubernetes /compute nova nova-api-wsgi write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}" if is_service_enabled ceilometer; then @@ -138,8 +111,6 @@ function create_nova_conf { # Set the oslo messaging driver to the typical default. This does not # enable notifications, but it will allow them to function when enabled. iniset $NOVA_CONF oslo_messaging_notifications driver "messagingv2" - iniset $NOVA_CONF oslo_messaging_notifications transport_url $(get_notification_url) - iniset $NOVA_CONF notifications notification_format "$NOVA_NOTIFICATION_FORMAT" kubernetes_ensure_resource secret/nova-cell1-rabbitmq NOVA_RABBITMQ_USERNAME=$(get_data_from_secret nova-cell1-rabbitmq openstack username) @@ -151,11 +122,6 @@ function create_nova_conf { # don't let the conductor get out of control now that we're using a pure python db driver iniset $NOVA_CONF conductor workers "$API_WORKERS" - if is_service_enabled tls-proxy; then - iniset $NOVA_CONF DEFAULT glance_protocol https - iniset $NOVA_CONF oslo_middleware enable_proxy_headers_parsing True - fi - iniset $NOVA_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT" if [ "$NOVA_USE_SERVICE_TOKEN" == "True" ]; then @@ -195,58 +161,51 @@ function create_nova_conf { # Console proxy configuration has to go after conductor configuration # because the per cell config file nova_cellN.conf is cleared out as part # of conductor configuration. - if [[ "${CELLSV2_SETUP}" == "singleconductor" ]]; then - configure_console_proxies - else - for i in $(seq 1 $NOVA_NUM_CELLS); do - local conf - local offset - conf=$(conductor_conf $i) - offset=$((i - 1)) - configure_console_proxies $conf $offset - done - fi + + configure_console_proxies + + # NOTE(Alex): Disable rootwrap and use sudo only + iniset $NOVA_CONF workarounds disable_rootwrap True } +function create_nova_compute_conf { + + # Bug #1802143: $NOVA_CPU_CONF is constructed by first copying $NOVA_CONF... + cp $NOVA_CONF $NOVA_CPU_CONF + # ...and then adding/overriding anything explicitly set in $NOVA_CPU_CONF + merge_config_file $TOP_DIR/local.conf post-config '$NOVA_CPU_CONF' + + if [[ "${CELLSV2_SETUP}" == "singleconductor" ]]; then + # NOTE(danms): Grenade doesn't setup multi-cell rabbit, so + # skip these bits and use the normal config. + echo "Skipping multi-cell conductor fleet setup" + else + # "${CELLSV2_SETUP}" is "superconductor" + # FIXME(danms): Should this be configurable? + iniset $NOVA_CPU_CONF workarounds disable_group_policy_check_upcall True + # Since the nova-compute service cannot reach nova-scheduler over + # RPC, we also disable track_instance_changes. + iniset $NOVA_CPU_CONF filter_scheduler track_instance_changes False + iniset_rpc_backend nova $NOVA_CPU_CONF DEFAULT "nova_cell${NOVA_CPU_CELL}" + fi + + # Make sure we nuke any database config + inidelete $NOVA_CPU_CONF database connection + inidelete $NOVA_CPU_CONF api_database connection + + # Console proxies were configured earlier in create_nova_conf. Now that the + # nova-cpu.conf has been created, configure the console settings required + # by the compute process. + configure_console_compute +} +export -f create_nova_compute_conf + function init_nova { + # All nova components talk to a central database. # Only do this step once on the API node for an entire cluster. - if is_service_enabled $DATABASE_BACKENDS && is_service_enabled n-api; then - kubernetes_ensure_resource service/nova-api-mysql-master - kubernetes_wait_pod_ready nova-api-mysql-0 - $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF api_db sync - - kubernetes_ensure_resource secret/nova-cell0-mysql - kubernetes_wait_pod_ready nova-cell0-mysql-0 - NOVA_CELL0_DATABASE_USER=$(get_data_from_secret nova-cell0-mysql openstack USER) - NOVA_CELL0_DATABASE_PASSWORD=$(get_data_from_secret nova-cell0-mysql openstack PASSWORD) - NOVA_CELL0_DATABASE_NAME=$(get_data_from_secret nova-cell0-mysql openstack DATABASE) - kubernetes_ensure_resource service/nova-cell0-mysql-master - $NOVA_BIN_DIR/nova-manage cell_v2 map_cell0 --database_connection "mysql+pymysql://$NOVA_CELL0_DATABASE_USER:$NOVA_CELL0_DATABASE_PASSWORD@nova-cell0-mysql-master.openstack.svc/$NOVA_CELL0_DATABASE_NAME?charset=utf8" - - # (Re)create nova databases - for i in $(seq 1 $NOVA_NUM_CELLS); do - $NOVA_BIN_DIR/nova-manage --config-file $(conductor_conf $i) db sync --local_cell - done - - # Migrate nova and nova_cell0 databases. - $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF db sync - - # Run online migrations on the new databases - # Needed for flavor conversion - $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF db online_data_migrations - - # create the cell1 cell for the main nova db where the hosts live - for i in $(seq 1 $NOVA_NUM_CELLS); do - $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF --config-file $(conductor_conf $i) cell_v2 create_cell --name "cell$i" - done - fi create_nova_keys_dir - - if [[ "$NOVA_BACKEND" == "LVM" ]]; then - init_default_lvm_volume_group - fi } @@ -254,4 +213,143 @@ function init_nova { function clean_iptables { echo noop } -export -f clean_iptables \ No newline at end of file +export -f clean_iptables + +# Configure access to placement from a nova service, usually +# compute, but sometimes conductor. +function configure_placement_nova_compute { + # Use the provided config file path or default to $NOVA_CONF. + + local conf=${1:-$NOVA_CONF} + kubernetes_ensure_resource secret/placement-application-credential + PLACEMENT_APPLICATION_CREDENTIAL_SECRET=$(get_data_from_secret placement-application-credential openstack secret) + PLACEMENT_APPLICATION_CREDENTIAL_ID=$(get_data_from_secret placement-application-credential openstack id) + iniset $conf placement auth_url $KEYSTONE_AUTH_URI_V3 + iniset $conf placement auth_type v3applicationcredential + iniset $conf placement application_credential_id $PLACEMENT_APPLICATION_CREDENTIAL_ID + iniset $conf placement application_credential_secret $PLACEMENT_APPLICATION_CREDENTIAL_SECRET +} + + +# Configure access to cinder. +function configure_cinder_access { + kubernetes_ensure_resource secret/cinder-application-credential + CINDER_APPLICATION_CREDENTIAL_SECRET=$(get_data_from_secret cinder-application-credential openstack secret) + CINDER_APPLICATION_CREDENTIAL_ID=$(get_data_from_secret cinder-application-credential openstack id) + iniset $NOVA_CONF cinder auth_url $KEYSTONE_AUTH_URI_V3 + iniset $NOVA_CONF cinder auth_type v3applicationcredential + iniset $NOVA_CONF cinder application_credential_id $CINDER_APPLICATION_CREDENTIAL_ID + iniset $NOVA_CONF cinder application_credential_secret $CINDER_APPLICATION_CREDENTIAL_SECRET +} + +# start_nova() - Start running processes +function start_nova_rest { + # Hack to set the path for rootwrap + local old_path=$PATH + export PATH=$NOVA_BIN_DIR:$PATH + + # nova-scheduler + kubernetes_ensure_resource ds/nova-scheduler + kubernetes_rollout_restart ds/nova-scheduler + kubernetes_rollout_status ds/nova-scheduler + + # nova-scheduler + kubernetes_ensure_resource ds/nova-metadata-api + kubernetes_rollout_restart ds/nova-metadata-api + kubernetes_rollout_status ds/nova-metadata-api + + export PATH=$old_path +} + +function start_nova_conductor { + kubernetes_ensure_resource ds/nova-conductor + kubernetes_rollout_restart ds/nova-conductor + kubernetes_rollout_status ds/nova-conductor +} + + +function start_nova_console_proxies { + # Hack to set the path for rootwrap + local old_path=$PATH + # This is needed to find the nova conf + export PATH=$NOVA_BIN_DIR:$PATH + + local api_cell_conf=$NOVA_CONF + + # novncproxy + kubernetes_ensure_resource ds/nova-novncproxy + kubernetes_rollout_restart ds/nova-novncproxy + kubernetes_rollout_status ds/nova-novncproxy + run_process n-xvnc "$NOVA_BIN_DIR/nova-xvpvncproxy --config-file $api_cell_conf" + run_process n-spice "$NOVA_BIN_DIR/nova-spicehtml5proxy --config-file $api_cell_conf --web $SPICE_WEB_DIR" + run_process n-sproxy "$NOVA_BIN_DIR/nova-serialproxy --config-file $api_cell_conf" + + export PATH=$old_path +} + +function configure_console_compute { + + # NOTE(Alex): Now imagine using one cell and novnc only, + # so no need to offset the proxy port. + + # Use the host IP instead of the service host because for multi-node, the + # service host will be the controller only. + local default_proxyclient_addr + default_proxyclient_addr=$(iniget $NOVA_CPU_CONF DEFAULT my_ip) + + NOVNCPROXY_URL=${NOVNCPROXY_URL:-"http://nova-novncproxy.openstack.svc/vnc_auto.html"} + iniset $NOVA_CPU_CONF vnc novncproxy_base_url "$NOVNCPROXY_URL" + + # Address on which instance vncservers will listen on compute hosts. + # For multi-host, this should be the management ip of the compute host. + VNCSERVER_LISTEN=${VNCSERVER_LISTEN:-$NOVA_SERVICE_LISTEN_ADDRESS} + VNCSERVER_PROXYCLIENT_ADDRESS=${VNCSERVER_PROXYCLIENT_ADDRESS:-$default_proxyclient_addr} + iniset $NOVA_CPU_CONF vnc server_listen "$VNCSERVER_LISTEN" + iniset $NOVA_CPU_CONF vnc server_proxyclient_address "$VNCSERVER_PROXYCLIENT_ADDRESS" +} + +function configure_console_proxies { + # Use the provided config file path or default to $NOVA_CONF. + local conf=${1:-$NOVA_CONF} + + # NOTE(Alex): Now imagine using one cell and novnc only, + # so no need to offset the proxy port. + iniset $conf vnc novncproxy_host "$NOVA_SERVICE_LISTEN_ADDRESS" + iniset $conf vnc novncproxy_port 6080 +} + +# start_nova_api() - Start the API process ahead of other things +function start_nova_api { + # Hack to set the path for rootwrap + local old_path=$PATH + export PATH=$NOVA_BIN_DIR:$PATH + kubernetes_ensure_resource ds/nova-compute-api + kubernetes_rollout_restart ds/nova-compute-api + kubernetes_rollout_status ds/nova-compute-api + export PATH=$old_path +} + +# start_nova_compute() - Start the compute process +function start_nova_compute { + # Hack to set the path for rootwrap + local old_path=$PATH + export PATH=$NOVA_BIN_DIR:$PATH + + # libvirt + kubernetes_ensure_resource ds/nova-compute + kubernetes_rollout_restart ds/nova-compute + kubernetes_rollout_status ds/nova-compute + + export PATH=$old_path +} + +# install_nova() - Collect source and prepare +# NOTE(Alex): Because, the nova stuff is in the stach.sh, +# keep nova installation on host.(kinda status check) + +function start_nova { + start_nova_rest + start_nova_console_proxies + start_nova_conductor + start_nova_compute +} \ No newline at end of file diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 6302be67..592978eb 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -51,8 +51,17 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then sudo chmod -Rv 777 /etc/ceph kubectl create secret generic ceph-config -n openstack \ --from-file=/etc/ceph/ceph.conf \ - --from-file=/etc/ceph/ceph.client.glance.keyring \ - --from-file=/etc/ceph/ceph.client.cinder.keyring + --from-file=/etc/ceph/ceph.client.cinder.keyring \ + --from-file=/etc/ceph/ceph.client.glance.keyring + + # NOTE(Alex): Create nova compute conf to include placement and libvirt config + create_nova_compute_conf + # NOTE(Alex) To include create_nova_conf_neutron and barbican hack config + kubectl create secret generic nova-config -n openstack \ + --from-file=/etc/nova/nova.conf \ + --from-file=/etc/nova/nova-cpu.conf \ + --from-file=/etc/nova/nova_cell1.conf \ + --from-file=/etc/nova/api-paste.ini elif [[ "$1" == "stack" && "$2" == "extra" ]]; then : diff --git a/images/nova/Dockerfile b/images/nova/Dockerfile new file mode 100644 index 00000000..1d460e2b --- /dev/null +++ b/images/nova/Dockerfile @@ -0,0 +1,54 @@ +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM vexxhost/python-builder as builder +FROM vexxhost/python-base AS nova-base +RUN mkdir -p /var/lib/nova && \ + chmod 777 -R /var/lib/nova + +FROM nova-base AS nova-compute-api +COPY nova-api-wsgi /usr/local/bin/nova-api-wsgi +EXPOSE 8774 +ENV UWSGI_HTTP_SOCKET=:8774 UWSGI_WSGI_FILE=/usr/local/bin/nova-api-wsgi +CMD ["/usr/local/bin/uwsgi", "--ini", "/etc/uwsgi/uwsgi.ini"] + +FROM nova-base AS nova-metadata-api +COPY nova-metadata-wsgi /usr/local/bin/nova-metadata-wsgi +EXPOSE 8775 +ENV UWSGI_HTTP_SOCKET=:8775 UWSGI_WSGI_FILE=/usr/local/bin/nova-metadata-wsgi +CMD ["/usr/local/bin/uwsgi", "--ini", "/etc/uwsgi/uwsgi.ini"] + +FROM nova-base AS nova-conductor +COPY nova-init /usr/local/bin/nova-init +COPY nova-conductor /usr/local/bin/nova-conductor +CMD ["/usr/local/bin/nova-conductor"] + +FROM nova-base AS nova-scheduler +COPY nova-scheduler /usr/local/bin/nova-scheduler +CMD ["/usr/local/bin/nova-scheduler"] + +FROM debian:10-slim AS novnc-builder +ADD https://github.com/novnc/noVNC/archive/v1.2.0.tar.gz novnc.tgz +RUN tar -xvzf novnc.tgz + +FROM nova-base AS nova-novncproxy +COPY nova-novncproxy /usr/local/bin/nova-novncproxy +COPY --from=novnc-builder noVNC-1.2.0/ /usr/share/novnc +EXPOSE 6080 +CMD ["/usr/local/bin/nova-novncproxy"] + +FROM nova-base AS nova-compute +COPY nova-compute /usr/local/bin/nova-compute +CMD ["/usr/local/bin/nova-compute"] \ No newline at end of file diff --git a/images/nova/bindep.txt b/images/nova/bindep.txt new file mode 100644 index 00000000..1da2cacb --- /dev/null +++ b/images/nova/bindep.txt @@ -0,0 +1,15 @@ +gcc [compile] +libc-dev [compile] +pkg-config [compile] +libvirt-dev [compile] +librados-dev [compile] +librbd-dev [compile] +librados2 +librbd1 +libvirt0 +ceph-common +genisoimage +open-iscsi +sudo +qemu-utils +openvswitch-common diff --git a/images/nova/build-requirements.txt b/images/nova/build-requirements.txt new file mode 100644 index 00000000..002d1b93 --- /dev/null +++ b/images/nova/build-requirements.txt @@ -0,0 +1 @@ +Cython diff --git a/images/nova/constraints.txt b/images/nova/constraints.txt new file mode 100644 index 00000000..e483a913 --- /dev/null +++ b/images/nova/constraints.txt @@ -0,0 +1 @@ +--constraint https://releases.openstack.org/constraints/upper/ussuri diff --git a/images/nova/nova-api-wsgi b/images/nova/nova-api-wsgi new file mode 100755 index 00000000..415f87c3 --- /dev/null +++ b/images/nova/nova-api-wsgi @@ -0,0 +1,28 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources + +import sentry_sdk + +from nova.api.openstack.compute.wsgi import init_application +from sentry_sdk.integrations import wsgi + +VERSION = pkg_resources.get_distribution("nova").version +sentry_sdk.init(release="nova@%s" % VERSION) + +application = init_application() +application = wsgi.SentryWsgiMiddleware(application) diff --git a/images/nova/nova-compute b/images/nova/nova-compute new file mode 100755 index 00000000..128d29af --- /dev/null +++ b/images/nova/nova-compute @@ -0,0 +1,29 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources +import re +import sys + +import sentry_sdk + +from nova.cmd.compute import main + +VERSION = pkg_resources.get_distribution("nova").version +sentry_sdk.init(release="nova@%s" % VERSION) + +sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) +sys.exit(main()) diff --git a/images/nova/nova-conductor b/images/nova/nova-conductor new file mode 100755 index 00000000..bdc7541a --- /dev/null +++ b/images/nova/nova-conductor @@ -0,0 +1,29 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources +import re +import sys + +import sentry_sdk + +from nova.cmd.conductor import main + +VERSION = pkg_resources.get_distribution("nova").version +sentry_sdk.init(release="nova@%s" % VERSION) + +sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) +sys.exit(main()) diff --git a/images/nova/nova-init b/images/nova/nova-init new file mode 100755 index 00000000..a83f961d --- /dev/null +++ b/images/nova/nova-init @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +nova-manage api_db sync +nova-manage cell_v2 map_cell0 --database_connection ${CELL0_DB} + +# NOTE(Alex) Use same commands in devstack +nova-manage --config-file /etc/nova/nova_cell1.conf db sync --local_cell +nova-manage --config-file /etc/nova/nova.conf db sync +nova-manage --config-file /etc/nova/nova.conf db online_data_migrations +nova-manage --config-file /etc/nova/nova.conf \ +--config-file /etc/nova/nova_cell1.conf \ +cell_v2 create_cell --name cell1 || error_code=$? + +if [ "${error_code}" -ne 2 ]; then + exit ${error_code} +fi diff --git a/images/nova/nova-metadata-wsgi b/images/nova/nova-metadata-wsgi new file mode 100755 index 00000000..424c0378 --- /dev/null +++ b/images/nova/nova-metadata-wsgi @@ -0,0 +1,28 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources + +import sentry_sdk + +from nova.api.metadata.wsgi import init_application +from sentry_sdk.integrations import wsgi + +VERSION = pkg_resources.get_distribution("nova").version +sentry_sdk.init(release="nova@%s" % VERSION) + +application = init_application() +application = wsgi.SentryWsgiMiddleware(application) diff --git a/images/nova/nova-novncproxy b/images/nova/nova-novncproxy new file mode 100755 index 00000000..188d2b15 --- /dev/null +++ b/images/nova/nova-novncproxy @@ -0,0 +1,29 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources +import re +import sys + +import sentry_sdk + +from nova.cmd.novncproxy import main + +VERSION = pkg_resources.get_distribution("nova").version +sentry_sdk.init(release="nova@%s" % VERSION) + +sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) +sys.exit(main()) diff --git a/images/nova/nova-scheduler b/images/nova/nova-scheduler new file mode 100755 index 00000000..b93d80c5 --- /dev/null +++ b/images/nova/nova-scheduler @@ -0,0 +1,29 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources +import re +import sys + +import sentry_sdk + +from nova.cmd.scheduler import main + +VERSION = pkg_resources.get_distribution("nova").version +sentry_sdk.init(release="nova@%s" % VERSION) + +sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) +sys.exit(main()) diff --git a/images/nova/requirements.txt b/images/nova/requirements.txt new file mode 100644 index 00000000..5b28554c --- /dev/null +++ b/images/nova/requirements.txt @@ -0,0 +1,8 @@ +uWSGI +PyMySQL +python-memcached +sentry-sdk +git+https://opendev.org/openstack/nova@stable/ussuri +libvirt-python +https://github.com/ceph/ceph/archive/v15.2.4.tar.gz#egg=rados&subdirectory=src/pybind/rados +https://github.com/ceph/ceph/archive/v15.2.4.tar.gz#egg=rbd&subdirectory=src/pybind/rbd \ No newline at end of file diff --git a/images/nova/setup-repos.sh b/images/nova/setup-repos.sh new file mode 100755 index 00000000..810a3781 --- /dev/null +++ b/images/nova/setup-repos.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +apt-get install -y gnupg2 + +cat <