# lib/neutron
# functions - functions specific to neutron

# Dependencies:
# ``functions`` file
# ``DEST`` must be defined
# ``STACK_USER`` must be defined

# ``stack.sh`` calls the entry points in this order:
#
# - install_neutron
# - install_neutronclient
# - install_neutron_agent_packages
# - install_neutron_third_party
# - configure_neutron
# - init_neutron
# - configure_neutron_third_party
# - init_neutron_third_party
# - start_neutron_third_party
# - create_neutron_cache_dir
# - create_nova_conf_neutron
# - start_neutron_service_and_check
# - start_neutron_agents
# - create_neutron_initial_network
# - setup_neutron_debug
#
# ``unstack.sh`` calls the entry points in this order:
#
# - stop_neutron
# - stop_neutron_third_party
# - cleanup_neutron

# Functions in lib/neutron are classified into the following categories:
#
# - entry points (called from stack.sh or unstack.sh)
# - internal functions
# - neutron exercises
# - 3rd party programs


# Neutron Networking
# ------------------

# Make sure that neutron is enabled in ``ENABLED_SERVICES``.  If you want
# to run Neutron on this host, make sure that q-svc is also in
# ``ENABLED_SERVICES``.
#
# See "Neutron Network Configuration" below for additional variables
# that must be set in localrc for connectivity across hosts with
# Neutron.
#
# With Neutron networking the NETWORK_MANAGER variable is ignored.

# Settings
# --------

# Timeout value in seconds to wait for IPv6 gateway configuration
GATEWAY_TIMEOUT=30


# Neutron Network Configuration
# -----------------------------

# Subnet IP version
IP_VERSION=${IP_VERSION:-4}
# Validate IP_VERSION
if [[ $IP_VERSION != "4" ]] && [[ $IP_VERSION != "6" ]] && [[ $IP_VERSION != "4+6" ]]; then
    die $LINENO "IP_VERSION must be either 4, 6, or 4+6"
fi
# Gateway and subnet defaults, in case they are not customized in localrc
NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1}
PUBLIC_NETWORK_GATEWAY=${PUBLIC_NETWORK_GATEWAY:-172.24.4.1}
PRIVATE_SUBNET_NAME=${PRIVATE_SUBNET_NAME:-"private-subnet"}
PUBLIC_SUBNET_NAME=${PUBLIC_SUBNET_NAME:-"public-subnet"}

if is_ssl_enabled_service "neutron" || is_service_enabled tls-proxy; then
    Q_PROTOCOL="https"
fi

# Generate 40-bit IPv6 Global ID to comply with RFC 4193
IPV6_GLOBAL_ID=`uuidgen | sed s/-//g | cut -c 23- | sed -e "s/\(..\)\(....\)\(....\)/\1:\2:\3/"`

# IPv6 gateway and subnet defaults, in case they are not customized in localrc
IPV6_RA_MODE=${IPV6_RA_MODE:-slaac}
IPV6_ADDRESS_MODE=${IPV6_ADDRESS_MODE:-slaac}
IPV6_PUBLIC_SUBNET_NAME=${IPV6_PUBLIC_SUBNET_NAME:-ipv6-public-subnet}
IPV6_PRIVATE_SUBNET_NAME=${IPV6_PRIVATE_SUBNET_NAME:-ipv6-private-subnet}
FIXED_RANGE_V6=${FIXED_RANGE_V6:-fd$IPV6_GLOBAL_ID::/64}
IPV6_PRIVATE_NETWORK_GATEWAY=${IPV6_PRIVATE_NETWORK_GATEWAY:-fd$IPV6_GLOBAL_ID::1}
IPV6_PUBLIC_RANGE=${IPV6_PUBLIC_RANGE:-fe80:cafe:cafe::/64}
IPV6_PUBLIC_NETWORK_GATEWAY=${IPV6_PUBLIC_NETWORK_GATEWAY:-fe80:cafe:cafe::2}
# IPV6_ROUTER_GW_IP must be defined when IP_VERSION=4+6 as it cannot be
# obtained conventionally until the l3-agent has support for dual-stack
# TODO (john-davidge) Remove once l3-agent supports dual-stack
IPV6_ROUTER_GW_IP=${IPV6_ROUTER_GW_IP:-fe80:cafe:cafe::1}

# Set up default directories
GITDIR["python-neutronclient"]=$DEST/python-neutronclient


NEUTRON_DIR=$DEST/neutron
NEUTRON_AUTH_CACHE_DIR=${NEUTRON_AUTH_CACHE_DIR:-/var/cache/neutron}

# Support entry points installation of console scripts
if [[ -d $NEUTRON_DIR/bin/neutron-server ]]; then
    NEUTRON_BIN_DIR=$NEUTRON_DIR/bin
else
    NEUTRON_BIN_DIR=$(get_python_exec_prefix)
fi

NEUTRON_CONF_DIR=/etc/neutron
NEUTRON_CONF=$NEUTRON_CONF_DIR/neutron.conf
export NEUTRON_TEST_CONFIG_FILE=${NEUTRON_TEST_CONFIG_FILE:-"$NEUTRON_CONF_DIR/debug.ini"}

# Agent binaries.  Note, binary paths for other agents are set in per-service
# scripts in lib/neutron_plugins/services/
AGENT_DHCP_BINARY="$NEUTRON_BIN_DIR/neutron-dhcp-agent"
AGENT_L3_BINARY=${AGENT_L3_BINARY:-"$NEUTRON_BIN_DIR/neutron-l3-agent"}
AGENT_META_BINARY="$NEUTRON_BIN_DIR/neutron-metadata-agent"

# Agent config files. Note, plugin-specific Q_PLUGIN_CONF_FILE is set and
# loaded from per-plugin  scripts in lib/neutron_plugins/
Q_DHCP_CONF_FILE=$NEUTRON_CONF_DIR/dhcp_agent.ini
Q_L3_CONF_FILE=$NEUTRON_CONF_DIR/l3_agent.ini
Q_FWAAS_CONF_FILE=$NEUTRON_CONF_DIR/fwaas_driver.ini
Q_VPN_CONF_FILE=$NEUTRON_CONF_DIR/vpn_agent.ini
Q_META_CONF_FILE=$NEUTRON_CONF_DIR/metadata_agent.ini

# Default name for Neutron database
Q_DB_NAME=${Q_DB_NAME:-neutron}
# Default Neutron Plugin
Q_PLUGIN=${Q_PLUGIN:-ml2}
# Default Neutron Port
Q_PORT=${Q_PORT:-9696}
# Default Neutron Internal Port when using TLS proxy
Q_PORT_INT=${Q_PORT_INT:-19696}
# Default Neutron Host
Q_HOST=${Q_HOST:-$SERVICE_HOST}
# Default protocol
Q_PROTOCOL=${Q_PROTOCOL:-$SERVICE_PROTOCOL}
# Default admin username
Q_ADMIN_USERNAME=${Q_ADMIN_USERNAME:-neutron}
# Default auth strategy
Q_AUTH_STRATEGY=${Q_AUTH_STRATEGY:-keystone}
# Use namespace or not
Q_USE_NAMESPACE=${Q_USE_NAMESPACE:-True}
# RHEL's support for namespaces requires using veths with ovs
Q_OVS_USE_VETH=${Q_OVS_USE_VETH:-False}
Q_USE_ROOTWRAP=${Q_USE_ROOTWRAP:-True}
# Meta data IP
Q_META_DATA_IP=${Q_META_DATA_IP:-$SERVICE_HOST}
# Allow Overlapping IP among subnets
Q_ALLOW_OVERLAPPING_IP=${Q_ALLOW_OVERLAPPING_IP:-True}
# Use neutron-debug command
Q_USE_DEBUG_COMMAND=${Q_USE_DEBUG_COMMAND:-False}
# The name of the default q-l3 router
Q_ROUTER_NAME=${Q_ROUTER_NAME:-router1}
# nova vif driver that all plugins should use
NOVA_VIF_DRIVER=${NOVA_VIF_DRIVER:-"nova.virt.libvirt.vif.LibvirtGenericVIFDriver"}
Q_NOTIFY_NOVA_PORT_STATUS_CHANGES=${Q_NOTIFY_NOVA_PORT_STATUS_CHANGES:-True}
Q_NOTIFY_NOVA_PORT_DATA_CHANGES=${Q_NOTIFY_NOVA_PORT_DATA_CHANGES:-True}
VIF_PLUGGING_IS_FATAL=${VIF_PLUGGING_IS_FATAL:-True}
VIF_PLUGGING_TIMEOUT=${VIF_PLUGGING_TIMEOUT:-300}
# Specify if the initial private and external networks should be created
NEUTRON_CREATE_INITIAL_NETWORKS=${NEUTRON_CREATE_INITIAL_NETWORKS:-True}

## Provider Network Information
PROVIDER_SUBNET_NAME=${PROVIDER_SUBNET_NAME:-"provider_net"}

# Use flat providernet for public network
#
# If Q_USE_PROVIDERNET_FOR_PUBLIC=True, use a flat provider network
# for external interface of neutron l3-agent.  In that case,
# PUBLIC_PHYSICAL_NETWORK specifies provider:physical_network value
# used for the network.  In case of ofagent, you should add the
# corresponding entry to your OFAGENT_PHYSICAL_INTERFACE_MAPPINGS.
# For openvswitch agent, you should add the corresponding entry to
# your OVS_BRIDGE_MAPPINGS.
#
# eg.  (ofagent)
#    Q_USE_PROVIDERNET_FOR_PUBLIC=True
#    Q_USE_PUBLIC_VETH=True
#    PUBLIC_PHYSICAL_NETWORK=public
#    OFAGENT_PHYSICAL_INTERFACE_MAPPINGS=public:veth-pub-int
#
# eg.  (openvswitch agent)
#    Q_USE_PROVIDERNET_FOR_PUBLIC=True
#    PUBLIC_PHYSICAL_NETWORK=public
#    OVS_BRIDGE_MAPPINGS=public:br-ex
Q_USE_PROVIDERNET_FOR_PUBLIC=${Q_USE_PROVIDERNET_FOR_PUBLIC:-False}
PUBLIC_PHYSICAL_NETWORK=${PUBLIC_PHYSICAL_NETWORK:-public}

# If Q_USE_PUBLIC_VETH=True, create and use a veth pair instead of
# PUBLIC_BRIDGE.  This is intended to be used with
# Q_USE_PROVIDERNET_FOR_PUBLIC=True.
Q_USE_PUBLIC_VETH=${Q_USE_PUBLIC_VETH:-False}
Q_PUBLIC_VETH_EX=${Q_PUBLIC_VETH_EX:-veth-pub-ex}
Q_PUBLIC_VETH_INT=${Q_PUBLIC_VETH_INT:-veth-pub-int}

# The next two variables are configured by plugin
# e.g.  _configure_neutron_l3_agent or lib/neutron_plugins/*
#
# The plugin supports L3.
Q_L3_ENABLED=${Q_L3_ENABLED:-False}
# L3 routers exist per tenant
Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-False}

# List of config file names in addition to the main plugin config file
# See _configure_neutron_common() for details about setting it up
declare -a Q_PLUGIN_EXTRA_CONF_FILES

# List of (optional) config files for VPN device drivers to use with
# the neutron-q-vpn agent
declare -a Q_VPN_EXTRA_CONF_FILES


Q_RR_CONF_FILE=$NEUTRON_CONF_DIR/rootwrap.conf
if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
    Q_RR_COMMAND="sudo"
else
    NEUTRON_ROOTWRAP=$(get_rootwrap_location neutron)
    Q_RR_COMMAND="sudo $NEUTRON_ROOTWRAP $Q_RR_CONF_FILE"
fi


# Distributed Virtual Router (DVR) configuration
# Can be:
# - ``legacy``   - No DVR functionality
# - ``dvr_snat`` - Controller or single node DVR
# - ``dvr``      - Compute node in multi-node DVR
#
Q_DVR_MODE=${Q_DVR_MODE:-legacy}
if [[ "$Q_DVR_MODE" != "legacy" ]]; then
    Q_ML2_PLUGIN_MECHANISM_DRIVERS=openvswitch,linuxbridge,l2population
fi

# Provider Network Configurations
# --------------------------------

# The following variables control the Neutron ML2 plugins' allocation
# of tenant networks and availability of provider networks. If these
# are not configured in ``localrc``, tenant networks will be local to
# the host (with no remote connectivity), and no physical resources
# will be available for the allocation of provider networks.

# To disable tunnels (GRE or VXLAN) for tenant networks,
# set to False in ``local.conf``.
# GRE tunnels are only supported by the openvswitch.
ENABLE_TENANT_TUNNELS=${ENABLE_TENANT_TUNNELS:-True}

# If using GRE tunnels for tenant networks, specify the range of
# tunnel IDs from which tenant networks are allocated. Can be
# overriden in ``localrc`` in necesssary.
TENANT_TUNNEL_RANGES=${TENANT_TUNNEL_RANGES:-1:1000}

# To use VLANs for tenant networks, set to True in localrc. VLANs
# are supported by the ML2 plugins, requiring additional configuration
# described below.
ENABLE_TENANT_VLANS=${ENABLE_TENANT_VLANS:-False}

# If using VLANs for tenant networks, set in ``localrc`` to specify
# the range of VLAN VIDs from which tenant networks are
# allocated. An external network switch must be configured to
# trunk these VLANs between hosts for multi-host connectivity.
#
# Example: ``TENANT_VLAN_RANGE=1000:1999``
TENANT_VLAN_RANGE=${TENANT_VLAN_RANGE:-}

# If using VLANs for tenant networks, or if using flat or VLAN
# provider networks, set in ``localrc`` to the name of the physical
# network, and also configure ``OVS_PHYSICAL_BRIDGE`` for the
# openvswitch agent or ``LB_PHYSICAL_INTERFACE`` for the linuxbridge
# agent, as described below.
#
# Example: ``PHYSICAL_NETWORK=default``
PHYSICAL_NETWORK=${PHYSICAL_NETWORK:-}

# With the openvswitch agent, if using VLANs for tenant networks,
# or if using flat or VLAN provider networks, set in ``localrc`` to
# the name of the OVS bridge to use for the physical network. The
# bridge will be created if it does not already exist, but a
# physical interface must be manually added to the bridge as a
# port for external connectivity.
#
# Example: ``OVS_PHYSICAL_BRIDGE=br-eth1``
OVS_PHYSICAL_BRIDGE=${OVS_PHYSICAL_BRIDGE:-}

# With the linuxbridge agent, if using VLANs for tenant networks,
# or if using flat or VLAN provider networks, set in ``localrc`` to
# the name of the network interface to use for the physical
# network.
#
# Example: ``LB_PHYSICAL_INTERFACE=eth1``
LB_PHYSICAL_INTERFACE=${LB_PHYSICAL_INTERFACE:-}

# When Neutron tunnels are enabled it is needed to specify the
# IP address of the end point in the local server. This IP is set
# by default to the same IP address that the HOST IP.
# This variable can be used to specify a different end point IP address
# Example: ``TUNNEL_ENDPOINT_IP=1.1.1.1``
TUNNEL_ENDPOINT_IP=${TUNNEL_ENDPOINT_IP:-$HOST_IP}

# With the openvswitch plugin, set to True in ``localrc`` to enable
# provider GRE tunnels when ``ENABLE_TENANT_TUNNELS`` is False.
#
# Example: ``OVS_ENABLE_TUNNELING=True``
OVS_ENABLE_TUNNELING=${OVS_ENABLE_TUNNELING:-$ENABLE_TENANT_TUNNELS}

# Use DHCP agent for providing metadata service in the case of
# without L3 agent (No Route Agent), set to True in localrc.
ENABLE_ISOLATED_METADATA=${ENABLE_ISOLATED_METADATA:-False}

# Add a static route as dhcp option, so the request to 169.254.169.254
# will be able to reach through a route(DHCP agent)
# This option require ENABLE_ISOLATED_METADATA = True
ENABLE_METADATA_NETWORK=${ENABLE_METADATA_NETWORK:-False}
# Neutron plugin specific functions
# ---------------------------------

# Please refer to ``lib/neutron_plugins/README.md`` for details.
source $TOP_DIR/lib/neutron_plugins/$Q_PLUGIN

# Agent loadbalancer service plugin functions
# -------------------------------------------

# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/loadbalancer

# Agent metering service plugin functions
# -------------------------------------------

# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/metering

# VPN service plugin functions
# -------------------------------------------
# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/vpn

# Firewall Service Plugin functions
# ---------------------------------
source $TOP_DIR/lib/neutron_plugins/services/firewall

# Use security group or not
if has_neutron_plugin_security_group; then
    Q_USE_SECGROUP=${Q_USE_SECGROUP:-True}
else
    Q_USE_SECGROUP=False
fi

# Tell Tempest this project is present
TEMPEST_SERVICES+=,neutron


# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace


# Functions
# ---------

function _determine_config_server {
    local cfg_file
    local opts="--config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE"
    for cfg_file in ${Q_PLUGIN_EXTRA_CONF_FILES[@]}; do
        opts+=" --config-file /$cfg_file"
    done
    echo "$opts"
}

function _determine_config_vpn {
    local cfg_file
    local opts="--config-file $NEUTRON_CONF --config-file=$Q_L3_CONF_FILE --config-file=$Q_VPN_CONF_FILE"
    if is_service_enabled q-fwaas; then
        opts+=" --config-file $Q_FWAAS_CONF_FILE"
    fi
    for cfg_file in ${Q_VPN_EXTRA_CONF_FILES[@]}; do
        opts+=" --config-file $cfg_file"
    done
    echo "$opts"

}

function _determine_config_l3 {
    local opts="--config-file $NEUTRON_CONF --config-file=$Q_L3_CONF_FILE"
    if is_service_enabled q-fwaas; then
        opts+=" --config-file $Q_FWAAS_CONF_FILE"
    fi
    echo "$opts"
}

# For services and agents that require it, dynamically construct a list of
# --config-file arguments that are passed to the binary.
function determine_config_files {
    local opts=""
    case "$1" in
        "neutron-server") opts="$(_determine_config_server)" ;;
        "neutron-vpn-agent") opts="$(_determine_config_vpn)" ;;
        "neutron-l3-agent") opts="$(_determine_config_l3)" ;;
    esac
    if [ -z "$opts" ] ; then
        die $LINENO "Could not determine config files for $1."
    fi
    echo "$opts"
}

# Test if any Neutron services are enabled
# is_neutron_enabled
function is_neutron_enabled {
    [[ ,${ENABLED_SERVICES} =~ ,"q-" ]] && return 0
    return 1
}

# configure_neutron()
# Set common config for all neutron server and agents.
function configure_neutron {
    _configure_neutron_common
    iniset_rpc_backend neutron $NEUTRON_CONF DEFAULT

    # goes before q-svc to init Q_SERVICE_PLUGIN_CLASSES
    if is_service_enabled q-lbaas; then
        _configure_neutron_lbaas
    fi
    if is_service_enabled q-metering; then
        _configure_neutron_metering
    fi
    if is_service_enabled q-vpn; then
        _configure_neutron_vpn
    fi
    if is_service_enabled q-fwaas; then
        _configure_neutron_fwaas
    fi
    if is_service_enabled q-agt q-svc; then
        _configure_neutron_service
    fi
    if is_service_enabled q-agt; then
        _configure_neutron_plugin_agent
    fi
    if is_service_enabled q-dhcp; then
        _configure_neutron_dhcp_agent
    fi
    if is_service_enabled q-l3; then
        _configure_neutron_l3_agent
    fi
    if is_service_enabled q-meta; then
        _configure_neutron_metadata_agent
    fi

    if [[ "$Q_DVR_MODE" != "legacy" ]]; then
        _configure_dvr
    fi
    if is_service_enabled ceilometer; then
        _configure_neutron_ceilometer_notifications
    fi

    _configure_neutron_debug_command
}

function create_nova_conf_neutron {
    iniset $NOVA_CONF DEFAULT network_api_class "nova.network.neutronv2.api.API"
    iniset $NOVA_CONF neutron admin_username "$Q_ADMIN_USERNAME"
    iniset $NOVA_CONF neutron admin_password "$SERVICE_PASSWORD"
    iniset $NOVA_CONF neutron admin_auth_url "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_AUTH_PORT/v2.0"
    iniset $NOVA_CONF neutron auth_strategy "$Q_AUTH_STRATEGY"
    iniset $NOVA_CONF neutron admin_tenant_name "$SERVICE_TENANT_NAME"
    iniset $NOVA_CONF neutron region_name "$REGION_NAME"
    iniset $NOVA_CONF neutron url "${Q_PROTOCOL}://$Q_HOST:$Q_PORT"

    if [[ "$Q_USE_SECGROUP" == "True" ]]; then
        LIBVIRT_FIREWALL_DRIVER=nova.virt.firewall.NoopFirewallDriver
        iniset $NOVA_CONF DEFAULT firewall_driver $LIBVIRT_FIREWALL_DRIVER
        iniset $NOVA_CONF DEFAULT security_group_api neutron
    fi

    # set NOVA_VIF_DRIVER and optionally set options in nova_conf
    neutron_plugin_create_nova_conf

    iniset $NOVA_CONF libvirt vif_driver "$NOVA_VIF_DRIVER"
    iniset $NOVA_CONF DEFAULT linuxnet_interface_driver "$LINUXNET_VIF_DRIVER"
    if is_service_enabled q-meta; then
        iniset $NOVA_CONF neutron service_metadata_proxy "True"
    fi

    iniset $NOVA_CONF DEFAULT vif_plugging_is_fatal "$VIF_PLUGGING_IS_FATAL"
    iniset $NOVA_CONF DEFAULT vif_plugging_timeout "$VIF_PLUGGING_TIMEOUT"
}

# create_neutron_cache_dir() - Part of the _neutron_setup_keystone() process
function create_neutron_cache_dir {
    # Create cache dir
    sudo mkdir -p $NEUTRON_AUTH_CACHE_DIR
    sudo chown $STACK_USER $NEUTRON_AUTH_CACHE_DIR
    rm -f $NEUTRON_AUTH_CACHE_DIR/*
}

# create_neutron_accounts() - Set up common required neutron accounts

# Tenant               User       Roles
# ------------------------------------------------------------------
# service              neutron    admin        # if enabled

# Migrated from keystone_data.sh
function create_neutron_accounts {

    local service_tenant=$(openstack project list | awk "/ $SERVICE_TENANT_NAME / { print \$2 }")
    local service_role=$(openstack role list | awk "/ service / { print \$2 }")

    if [[ "$ENABLED_SERVICES" =~ "q-svc" ]]; then

        local neutron_user=$(get_or_create_user "neutron" \
            "$SERVICE_PASSWORD" $service_tenant)
        get_or_add_user_role $service_role $neutron_user $service_tenant

        if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then

            local neutron_service=$(get_or_create_service "neutron" \
                "network" "Neutron Service")
            get_or_create_endpoint $neutron_service \
                "$REGION_NAME" \
                "$Q_PROTOCOL://$SERVICE_HOST:$Q_PORT/" \
                "$Q_PROTOCOL://$SERVICE_HOST:$Q_PORT/" \
                "$Q_PROTOCOL://$SERVICE_HOST:$Q_PORT/"
        fi
    fi
}

function create_neutron_initial_network {
    TENANT_ID=$(openstack project list | grep " demo " | get_field 1)
    die_if_not_set $LINENO TENANT_ID "Failure retrieving TENANT_ID for demo"

    if is_provider_network; then
        die_if_not_set $LINENO PHYSICAL_NETWORK "You must specify the PHYSICAL_NETWORK"
        die_if_not_set $LINENO PROVIDER_NETWORK_TYPE "You must specifiy the PROVIDER_NETWORK_TYPE"
        NET_ID=$(neutron net-create $PHYSICAL_NETWORK --tenant_id $TENANT_ID --provider:network_type $PROVIDER_NETWORK_TYPE --provider:physical_network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider:segmentation_id $SEGMENTATION_ID} --shared | grep ' id ' | get_field 2)
        SUBNET_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} --name $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY $NET_ID $FIXED_RANGE | grep ' id ' | get_field 2)
        SUBNET_V6_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 6 --ipv6-address-mode slaac --gateway $V6_NETWORK_GATEWAY --name $PROVIDER_SUBNET_NAME_V6 $NET_ID $FIXED_RANGE_V6 | grep 'id' | get_field 2)
        sudo ip link set $OVS_PHYSICAL_BRIDGE up
        sudo ip link set br-int up
        sudo ip link set $PUBLIC_INTERFACE up
    else
        NET_ID=$(neutron net-create --tenant-id $TENANT_ID "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $TENANT_ID"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            # Create IPv4 private subnet
            SUBNET_ID=$(_neutron_create_private_subnet_v4)
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            # Create IPv6 private subnet
            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6)
        fi
    fi

    if [[ "$Q_L3_ENABLED" == "True" ]]; then
        # Create a router, and add the private subnet as one of its interfaces
        if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then
            # create a tenant-owned router.
            ROUTER_ID=$(neutron router-create --tenant-id $TENANT_ID $Q_ROUTER_NAME | grep ' id ' | get_field 2)
            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $TENANT_ID $Q_ROUTER_NAME"
        else
            # Plugin only supports creating a single router, which should be admin owned.
            ROUTER_ID=$(neutron router-create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $Q_ROUTER_NAME"
        fi

        # Create an external network, and a subnet. Configure the external network as router gw
        if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" = "True" ]; then
            EXT_NET_ID=$(neutron net-create "$PUBLIC_NETWORK_NAME" -- --router:external=True --provider:network_type=flat --provider:physical_network=${PUBLIC_PHYSICAL_NETWORK} | grep ' id ' | get_field 2)
        else
            EXT_NET_ID=$(neutron net-create "$PUBLIC_NETWORK_NAME" -- --router:external=True | grep ' id ' | get_field 2)
        fi
        die_if_not_set $LINENO EXT_NET_ID "Failure creating EXT_NET_ID for $PUBLIC_NETWORK_NAME"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            # Configure router for IPv4 public access
            _neutron_configure_router_v4
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            # Configure router for IPv6 public access
            _neutron_configure_router_v6
        fi
    fi
}

# init_neutron() - Initialize databases, etc.
function init_neutron {
    recreate_database $Q_DB_NAME utf8
    # Run Neutron db migrations
    $NEUTRON_BIN_DIR/neutron-db-manage --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE upgrade head
}

# install_neutron() - Collect source and prepare
function install_neutron {
    git_clone $NEUTRON_REPO $NEUTRON_DIR $NEUTRON_BRANCH
    setup_develop $NEUTRON_DIR

    if [ "$VIRT_DRIVER" == 'xenserver' ]; then
        local dom0_ip
        dom0_ip=$(echo "$XENAPI_CONNECTION_URL" | cut -d "/" -f 3-)

        local ssh_dom0
        ssh_dom0="sudo -u $DOMZERO_USER ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$dom0_ip"

        # Find where the plugins should go in dom0
        local xen_functions
        xen_functions=$(cat $TOP_DIR/tools/xen/functions)
        local plugin_dir
        plugin_dir=$($ssh_dom0 "$xen_functions; set -eux; xapi_plugin_location")

        # install neutron plugins to dom0
        tar -czf - -C $NEUTRON_DIR/neutron/plugins/openvswitch/agent/xenapi/etc/xapi.d/plugins/ ./ |
            $ssh_dom0 "tar -xzf - -C $plugin_dir && chmod a+x $plugin_dir/*"
    fi
}

# install_neutronclient() - Collect source and prepare
function install_neutronclient {
    if use_library_from_git "python-neutronclient"; then
        git_clone_by_name "python-neutronclient"
        setup_dev_lib "python-neutronclient"
        sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-neutronclient"]}/tools/,/etc/bash_completion.d/}neutron.bash_completion
    fi
}

# install_neutron_agent_packages() - Collect source and prepare
function install_neutron_agent_packages {
    # radvd doesn't come with the OS. Install it if the l3 service is enabled.
    if is_service_enabled q-l3; then
        install_package radvd
    fi
    # install packages that are specific to plugin agent(s)
    if is_service_enabled q-agt q-dhcp q-l3; then
        neutron_plugin_install_agent_packages
    fi

    if is_service_enabled q-lbaas; then
        neutron_agent_lbaas_install_agent_packages
    fi
}

# Start running processes, including screen
function start_neutron_service_and_check {
    local cfg_file_options="$(determine_config_files neutron-server)"
    local service_port=$Q_PORT
    local service_protocol=$Q_PROTOCOL
    if is_service_enabled tls-proxy; then
        service_port=$Q_PORT_INT
        service_protocol="http"
    fi
    # Start the Neutron service
    run_process q-svc "python $NEUTRON_BIN_DIR/neutron-server $cfg_file_options"
    echo "Waiting for Neutron to start..."
    if is_ssl_enabled_service "neutron"; then
        ssl_ca="--ca-certificate=${SSL_BUNDLE_FILE}"
    fi
    if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget ${ssl_ca} --no-proxy -q -O- $service_protocol://$Q_HOST:$service_port; do sleep 1; done"; then
        die $LINENO "Neutron did not start"
    fi
    # Start proxy if enabled
    if is_service_enabled tls-proxy; then
        start_tls_proxy '*' $Q_PORT $Q_HOST $Q_PORT_INT &
    fi
}

# Start running processes, including screen
function start_neutron_agents {
    # Start up the neutron agents if enabled
    run_process q-agt "python $AGENT_BINARY --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE"
    run_process q-dhcp "python $AGENT_DHCP_BINARY --config-file $NEUTRON_CONF --config-file=$Q_DHCP_CONF_FILE"

    if is_provider_network; then
        sudo ovs-vsctl add-port $OVS_PHYSICAL_BRIDGE $PUBLIC_INTERFACE
        sudo ip link set $OVS_PHYSICAL_BRIDGE up
        sudo ip link set br-int up
        sudo ip link set $PUBLIC_INTERFACE up
        if is_ironic_hardware; then
            for IP in $(ip addr show dev $PUBLIC_INTERFACE | grep ' inet ' | awk '{print $2}'); do
                sudo ip addr del $IP dev $PUBLIC_INTERFACE
                sudo ip addr add $IP dev $OVS_PHYSICAL_BRIDGE
            done
            sudo route add -net $FIXED_RANGE gw $NETWORK_GATEWAY dev $OVS_PHYSICAL_BRIDGE
        fi
    fi

    if is_service_enabled q-vpn; then
        run_process q-vpn "$AGENT_VPN_BINARY $(determine_config_files neutron-vpn-agent)"
    else
        run_process q-l3 "python $AGENT_L3_BINARY $(determine_config_files neutron-l3-agent)"
    fi

    run_process q-meta "python $AGENT_META_BINARY --config-file $NEUTRON_CONF --config-file=$Q_META_CONF_FILE"

    if [ "$VIRT_DRIVER" = 'xenserver' ]; then
        # For XenServer, start an agent for the domU openvswitch
        run_process q-domua "python $AGENT_BINARY --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE.domU"
    fi

    if is_service_enabled q-lbaas; then
        run_process q-lbaas "python $AGENT_LBAAS_BINARY --config-file $NEUTRON_CONF --config-file=$LBAAS_AGENT_CONF_FILENAME"
    fi

    if is_service_enabled q-metering; then
        run_process q-metering "python $AGENT_METERING_BINARY --config-file $NEUTRON_CONF --config-file $METERING_AGENT_CONF_FILENAME"
    fi
}

# stop_neutron() - Stop running processes (non-screen)
function stop_neutron {
    if is_service_enabled q-dhcp; then
        pid=$(ps aux | awk '/[d]nsmasq.+interface=(tap|ns-)/ { print $2 }')
        [ ! -z "$pid" ] && sudo kill -9 $pid
    fi
    if is_service_enabled q-meta; then
        sudo pkill -9 -f neutron-ns-metadata-proxy || :
    fi

    if is_service_enabled q-lbaas; then
        neutron_lbaas_stop
    fi
    if is_service_enabled q-fwaas; then
        neutron_fwaas_stop
    fi
    if is_service_enabled q-vpn; then
        neutron_vpn_stop
    fi
    if is_service_enabled q-metering; then
        neutron_metering_stop
    fi
}

# cleanup_neutron() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_neutron {
    if [[ is_provider_network && is_ironic_hardware ]]; then
        for IP in $(ip addr show dev $OVS_PHYSICAL_BRIDGE | grep ' inet ' | awk '{print $2}'); do
            sudo ip addr del $IP dev $OVS_PHYSICAL_BRIDGE
            sudo ip addr add $IP dev $PUBLIC_INTERFACE
        done
        sudo route del -net $FIXED_RANGE gw $NETWORK_GATEWAY dev $OVS_PHYSICAL_BRIDGE
    fi

    if is_neutron_ovs_base_plugin; then
        neutron_ovs_base_cleanup
    fi

    # delete all namespaces created by neutron
    for ns in $(sudo ip netns list | grep -o -E '(qdhcp|qrouter|qlbaas|fip|snat)-[0-9a-f-]*'); do
        sudo ip netns delete ${ns}
    done
}

# _configure_neutron_common()
# Set common config for all neutron server and agents.
# This MUST be called before other ``_configure_neutron_*`` functions.
function _configure_neutron_common {
    # Put config files in ``NEUTRON_CONF_DIR`` for everyone to find
    if [[ ! -d $NEUTRON_CONF_DIR ]]; then
        sudo mkdir -p $NEUTRON_CONF_DIR
    fi
    sudo chown $STACK_USER $NEUTRON_CONF_DIR

    cp $NEUTRON_DIR/etc/neutron.conf $NEUTRON_CONF

    # Set plugin-specific variables ``Q_DB_NAME``, ``Q_PLUGIN_CLASS``.
    # For main plugin config file, set ``Q_PLUGIN_CONF_PATH``, ``Q_PLUGIN_CONF_FILENAME``.
    # For addition plugin config files, set ``Q_PLUGIN_EXTRA_CONF_PATH``,
    # ``Q_PLUGIN_EXTRA_CONF_FILES``.  For example:
    #
    #    ``Q_PLUGIN_EXTRA_CONF_FILES=(file1, file2)``
    neutron_plugin_configure_common

    if [[ "$Q_PLUGIN_CONF_PATH" == '' || "$Q_PLUGIN_CONF_FILENAME" == '' || "$Q_PLUGIN_CLASS" == '' ]]; then
        die $LINENO "Neutron plugin not set.. exiting"
    fi

    # If needed, move config file from ``$NEUTRON_DIR/etc/neutron`` to ``NEUTRON_CONF_DIR``
    mkdir -p /$Q_PLUGIN_CONF_PATH
    Q_PLUGIN_CONF_FILE=$Q_PLUGIN_CONF_PATH/$Q_PLUGIN_CONF_FILENAME
    cp $NEUTRON_DIR/$Q_PLUGIN_CONF_FILE /$Q_PLUGIN_CONF_FILE

    iniset $NEUTRON_CONF database connection `database_connection_url $Q_DB_NAME`
    iniset $NEUTRON_CONF DEFAULT state_path $DATA_DIR/neutron
    iniset $NEUTRON_CONF DEFAULT use_syslog $SYSLOG
    # If addition config files are set, make sure their path name is set as well
    if [[ ${#Q_PLUGIN_EXTRA_CONF_FILES[@]} > 0 && $Q_PLUGIN_EXTRA_CONF_PATH == '' ]]; then
        die $LINENO "Neutron additional plugin config not set.. exiting"
    fi

    # If additional config files exist, copy them over to neutron configuration
    # directory
    if [[ $Q_PLUGIN_EXTRA_CONF_PATH != '' ]]; then
        local f
        for (( f=0; $f < ${#Q_PLUGIN_EXTRA_CONF_FILES[@]}; f+=1 )); do
            Q_PLUGIN_EXTRA_CONF_FILES[$f]=$Q_PLUGIN_EXTRA_CONF_PATH/${Q_PLUGIN_EXTRA_CONF_FILES[$f]}
        done
    fi

    if [ "$VIRT_DRIVER" = 'fake' ]; then
        # Disable arbitrary limits
        iniset $NEUTRON_CONF quotas quota_network -1
        iniset $NEUTRON_CONF quotas quota_subnet -1
        iniset $NEUTRON_CONF quotas quota_port -1
        iniset $NEUTRON_CONF quotas quota_security_group -1
        iniset $NEUTRON_CONF quotas quota_security_group_rule -1
    fi

    # Format logging
    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
        setup_colorized_logging $NEUTRON_CONF DEFAULT project_id
    fi

    if is_service_enabled tls-proxy; then
        # Set the service port for a proxy to take the original
        iniset $NEUTRON_CONF DEFAULT bind_port "$Q_PORT_INT"
    fi

    if is_ssl_enabled_service "nova"; then
        iniset $NEUTRON_CONF DEFAULT nova_ca_certificates_file "$SSL_BUNDLE_FILE"
    fi

    if is_ssl_enabled_service "neutron"; then
        ensure_certificates NEUTRON

        iniset $NEUTRON_CONF DEFAULT use_ssl True
        iniset $NEUTRON_CONF DEFAULT ssl_cert_file "$NEUTRON_SSL_CERT"
        iniset $NEUTRON_CONF DEFAULT ssl_key_file "$NEUTRON_SSL_KEY"
    fi

    _neutron_setup_rootwrap
}

function _configure_neutron_debug_command {
    if [[ "$Q_USE_DEBUG_COMMAND" != "True" ]]; then
        return
    fi

    cp $NEUTRON_DIR/etc/l3_agent.ini $NEUTRON_TEST_CONFIG_FILE

    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT verbose False
    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT debug False
    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT use_namespaces $Q_USE_NAMESPACE
    iniset $NEUTRON_TEST_CONFIG_FILE agent root_helper "$Q_RR_COMMAND"

    _neutron_setup_interface_driver $NEUTRON_TEST_CONFIG_FILE

    neutron_plugin_configure_debug_command
}

function _configure_neutron_dhcp_agent {

    cp $NEUTRON_DIR/etc/dhcp_agent.ini $Q_DHCP_CONF_FILE

    iniset $Q_DHCP_CONF_FILE DEFAULT verbose True
    iniset $Q_DHCP_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_DHCP_CONF_FILE DEFAULT use_namespaces $Q_USE_NAMESPACE
    iniset $Q_DHCP_CONF_FILE DEFAULT root_helper "$Q_RR_COMMAND"

    if ! is_service_enabled q-l3; then
        if [[ "$ENABLE_ISOLATED_METADATA" = "True" ]]; then
            iniset $Q_DHCP_CONF_FILE DEFAULT enable_isolated_metadata $ENABLE_ISOLATED_METADATA
            iniset $Q_DHCP_CONF_FILE DEFAULT enable_metadata_network $ENABLE_METADATA_NETWORK
        else
            if [[ "$ENABLE_METADATA_NETWORK" = "True" ]]; then
                die "$LINENO" "Enable isolated metadata is a must for metadata network"
            fi
        fi
    fi

    _neutron_setup_interface_driver $Q_DHCP_CONF_FILE

    neutron_plugin_configure_dhcp_agent
}

function _configure_neutron_l3_agent {
    local cfg_file
    Q_L3_ENABLED=True
    # for l3-agent, only use per tenant router if we have namespaces
    Q_L3_ROUTER_PER_TENANT=$Q_USE_NAMESPACE

    if is_service_enabled q-vpn; then
        cp $NEUTRON_DIR/etc/vpn_agent.ini $Q_VPN_CONF_FILE
    fi

    cp $NEUTRON_DIR/etc/l3_agent.ini $Q_L3_CONF_FILE

    iniset $Q_L3_CONF_FILE DEFAULT verbose True
    iniset $Q_L3_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_L3_CONF_FILE DEFAULT use_namespaces $Q_USE_NAMESPACE
    iniset $Q_L3_CONF_FILE DEFAULT root_helper "$Q_RR_COMMAND"

    _neutron_setup_interface_driver $Q_L3_CONF_FILE

    neutron_plugin_configure_l3_agent
}

function _configure_neutron_metadata_agent {
    cp $NEUTRON_DIR/etc/metadata_agent.ini $Q_META_CONF_FILE

    iniset $Q_META_CONF_FILE DEFAULT verbose True
    iniset $Q_META_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_META_CONF_FILE DEFAULT nova_metadata_ip $Q_META_DATA_IP
    iniset $Q_META_CONF_FILE DEFAULT root_helper "$Q_RR_COMMAND"

    _neutron_setup_keystone $Q_META_CONF_FILE DEFAULT

}

function _configure_neutron_ceilometer_notifications {
    iniset $NEUTRON_CONF DEFAULT notification_driver messaging
}

function _configure_neutron_lbaas {
    neutron_agent_lbaas_configure_common
    neutron_agent_lbaas_configure_agent
}

function _configure_neutron_metering {
    neutron_agent_metering_configure_common
    neutron_agent_metering_configure_agent
}

function _configure_neutron_fwaas {
    neutron_fwaas_configure_common
    neutron_fwaas_configure_driver
}

function _configure_neutron_vpn {
    neutron_vpn_install_agent_packages
    neutron_vpn_configure_common
}

function _configure_dvr {
    iniset $NEUTRON_CONF DEFAULT router_distributed True
    iniset $Q_L3_CONF_FILE DEFAULT agent_mode $Q_DVR_MODE
}


# _configure_neutron_plugin_agent() - Set config files for neutron plugin agent
# It is called when q-agt is enabled.
function _configure_neutron_plugin_agent {
    # Specify the default root helper prior to agent configuration to
    # ensure that an agent's configuration can override the default
    iniset /$Q_PLUGIN_CONF_FILE agent root_helper "$Q_RR_COMMAND"
    iniset $NEUTRON_CONF DEFAULT verbose True
    iniset $NEUTRON_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

    # Configure agent for plugin
    neutron_plugin_configure_plugin_agent
}

# _configure_neutron_service() - Set config files for neutron service
# It is called when q-svc is enabled.
function _configure_neutron_service {
    Q_API_PASTE_FILE=$NEUTRON_CONF_DIR/api-paste.ini
    Q_POLICY_FILE=$NEUTRON_CONF_DIR/policy.json

    cp $NEUTRON_DIR/etc/api-paste.ini $Q_API_PASTE_FILE
    cp $NEUTRON_DIR/etc/policy.json $Q_POLICY_FILE

    # allow neutron user to administer neutron to match neutron account
    sed -i 's/"context_is_admin":  "role:admin"/"context_is_admin":  "role:admin or user_name:neutron"/g' $Q_POLICY_FILE

    # Update either configuration file with plugin
    iniset $NEUTRON_CONF DEFAULT core_plugin $Q_PLUGIN_CLASS

    if [[ $Q_SERVICE_PLUGIN_CLASSES != '' ]]; then
        iniset $NEUTRON_CONF DEFAULT service_plugins $Q_SERVICE_PLUGIN_CLASSES
    fi

    iniset $NEUTRON_CONF DEFAULT verbose True
    iniset $NEUTRON_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $NEUTRON_CONF DEFAULT policy_file $Q_POLICY_FILE
    iniset $NEUTRON_CONF DEFAULT allow_overlapping_ips $Q_ALLOW_OVERLAPPING_IP

    iniset $NEUTRON_CONF DEFAULT auth_strategy $Q_AUTH_STRATEGY
    _neutron_setup_keystone $NEUTRON_CONF keystone_authtoken

    # Configuration for neutron notifations to nova.
    iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_status_changes $Q_NOTIFY_NOVA_PORT_STATUS_CHANGES
    iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_data_changes $Q_NOTIFY_NOVA_PORT_DATA_CHANGES
    iniset $NEUTRON_CONF DEFAULT nova_url "$NOVA_SERVICE_PROTOCOL://$NOVA_SERVICE_HOST:$NOVA_SERVICE_PORT/v2"
    iniset $NEUTRON_CONF DEFAULT nova_admin_username nova
    iniset $NEUTRON_CONF DEFAULT nova_admin_password $SERVICE_PASSWORD
    ADMIN_TENANT_ID=$(openstack project list | awk "/ service / { print \$2 }")
    iniset $NEUTRON_CONF DEFAULT nova_admin_tenant_id $ADMIN_TENANT_ID
    iniset $NEUTRON_CONF DEFAULT nova_admin_auth_url  "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_AUTH_PORT/v2.0"

    # Configure plugin
    neutron_plugin_configure_service
}

# Utility Functions
#------------------

# _neutron_service_plugin_class_add() - add service plugin class
function _neutron_service_plugin_class_add {
    local service_plugin_class=$1
    if [[ $Q_SERVICE_PLUGIN_CLASSES == '' ]]; then
        Q_SERVICE_PLUGIN_CLASSES=$service_plugin_class
    elif [[ ! ,${Q_SERVICE_PLUGIN_CLASSES}, =~ ,${service_plugin_class}, ]]; then
        Q_SERVICE_PLUGIN_CLASSES="$Q_SERVICE_PLUGIN_CLASSES,$service_plugin_class"
    fi
}

# _neutron_setup_rootwrap() - configure Neutron's rootwrap
function _neutron_setup_rootwrap {
    if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
        return
    fi
    # Deploy new rootwrap filters files (owned by root).
    # Wipe any existing ``rootwrap.d`` files first
    Q_CONF_ROOTWRAP_D=$NEUTRON_CONF_DIR/rootwrap.d
    if [[ -d $Q_CONF_ROOTWRAP_D ]]; then
        sudo rm -rf $Q_CONF_ROOTWRAP_D
    fi
    # Deploy filters to ``$NEUTRON_CONF_DIR/rootwrap.d``
    mkdir -p -m 755 $Q_CONF_ROOTWRAP_D
    cp -pr $NEUTRON_DIR/etc/neutron/rootwrap.d/* $Q_CONF_ROOTWRAP_D/
    sudo chown -R root:root $Q_CONF_ROOTWRAP_D
    sudo chmod 644 $Q_CONF_ROOTWRAP_D/*
    # Set up ``rootwrap.conf``, pointing to ``$NEUTRON_CONF_DIR/rootwrap.d``
    # location moved in newer versions, prefer new location
    if test -r $NEUTRON_DIR/etc/neutron/rootwrap.conf; then
        sudo cp -p $NEUTRON_DIR/etc/neutron/rootwrap.conf $Q_RR_CONF_FILE
    else
        sudo cp -p $NEUTRON_DIR/etc/rootwrap.conf $Q_RR_CONF_FILE
    fi
    sudo sed -e "s:^filters_path=.*$:filters_path=$Q_CONF_ROOTWRAP_D:" -i $Q_RR_CONF_FILE
    sudo chown root:root $Q_RR_CONF_FILE
    sudo chmod 0644 $Q_RR_CONF_FILE
    # Specify ``rootwrap.conf`` as first parameter to neutron-rootwrap
    ROOTWRAP_SUDOER_CMD="$NEUTRON_ROOTWRAP $Q_RR_CONF_FILE *"

    # Set up the rootwrap sudoers for neutron
    TEMPFILE=`mktemp`
    echo "$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_SUDOER_CMD" >$TEMPFILE
    chmod 0440 $TEMPFILE
    sudo chown root:root $TEMPFILE
    sudo mv $TEMPFILE /etc/sudoers.d/neutron-rootwrap

    # Update the root_helper
    iniset $NEUTRON_CONF agent root_helper "$Q_RR_COMMAND"
}

# Configures keystone integration for neutron service and agents
function _neutron_setup_keystone {
    local conf_file=$1
    local section=$2

    create_neutron_cache_dir
    configure_auth_token_middleware $conf_file $Q_ADMIN_USERNAME $NEUTRON_AUTH_CACHE_DIR $section
}

function _neutron_setup_interface_driver {

    # ovs_use_veth needs to be set before the plugin configuration
    # occurs to allow plugins to override the setting.
    iniset $1 DEFAULT ovs_use_veth $Q_OVS_USE_VETH

    neutron_plugin_setup_interface_driver $1
}

# Create private IPv4 subnet
function _neutron_create_private_subnet_v4 {
    local subnet_params="--tenant-id $TENANT_ID "
    subnet_params+="--ip_version 4 "
    subnet_params+="--gateway $NETWORK_GATEWAY "
    subnet_params+="--name $PRIVATE_SUBNET_NAME "
    subnet_params+="$NET_ID $FIXED_RANGE"
    local subnet_id=$(neutron subnet-create $subnet_params | grep ' id ' | get_field 2)
    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet for $TENANT_ID"
    echo $subnet_id
}

# Create private IPv6 subnet
function _neutron_create_private_subnet_v6 {
    die_if_not_set $LINENO IPV6_RA_MODE "IPV6 RA Mode not set"
    die_if_not_set $LINENO IPV6_ADDRESS_MODE "IPV6 Address Mode not set"
    local ipv6_modes="--ipv6-ra-mode $IPV6_RA_MODE --ipv6-address-mode $IPV6_ADDRESS_MODE"
    local subnet_params="--tenant-id $TENANT_ID "
    subnet_params+="--ip_version 6 "
    subnet_params+="--gateway $IPV6_PRIVATE_NETWORK_GATEWAY "
    subnet_params+="--name $IPV6_PRIVATE_SUBNET_NAME "
    subnet_params+="$NET_ID $FIXED_RANGE_V6 $ipv6_modes"
    local ipv6_subnet_id=$(neutron subnet-create $subnet_params | grep ' id ' | get_field 2)
    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet for $TENANT_ID"
    echo $ipv6_subnet_id
}

# Create public IPv4 subnet
function _neutron_create_public_subnet_v4 {
    local subnet_params+="--ip_version 4 "
    subnet_params+="${Q_FLOATING_ALLOCATION_POOL:+--allocation-pool $Q_FLOATING_ALLOCATION_POOL} "
    subnet_params+="--gateway $PUBLIC_NETWORK_GATEWAY "
    subnet_params+="--name $PUBLIC_SUBNET_NAME "
    subnet_params+="$EXT_NET_ID $FLOATING_RANGE "
    subnet_params+="-- --enable_dhcp=False"
    local id_and_ext_gw_ip=$(neutron subnet-create $subnet_params | grep -e 'gateway_ip' -e ' id ')
    die_if_not_set $LINENO id_and_ext_gw_ip "Failure creating public IPv4 subnet"
    echo $id_and_ext_gw_ip
}

# Create public IPv6 subnet
function _neutron_create_public_subnet_v6 {
    local subnet_params="--ip_version 6 "
    subnet_params+="--gateway $IPV6_PUBLIC_NETWORK_GATEWAY "
    subnet_params+="--name $IPV6_PUBLIC_SUBNET_NAME "
    subnet_params+="$EXT_NET_ID $IPV6_PUBLIC_RANGE "
    subnet_params+="-- --enable_dhcp=False"
    local ipv6_id_and_ext_gw_ip=$(neutron subnet-create $subnet_params | grep -e 'gateway_ip' -e ' id ')
    die_if_not_set $LINENO ipv6_id_and_ext_gw_ip "Failure creating an IPv6 public subnet"
    echo $ipv6_id_and_ext_gw_ip
}

# Configure neutron router for IPv4 public access
function _neutron_configure_router_v4 {
    neutron router-interface-add $ROUTER_ID $SUBNET_ID
    # Create a public subnet on the external network
    local id_and_ext_gw_ip=$(_neutron_create_public_subnet_v4 $EXT_NET_ID)
    local ext_gw_ip=$(echo $id_and_ext_gw_ip  | get_field 2)
    PUB_SUBNET_ID=$(echo $id_and_ext_gw_ip | get_field 5)
    # Configure the external network as the default router gateway
    neutron router-gateway-set $ROUTER_ID $EXT_NET_ID

    # This logic is specific to using the l3-agent for layer 3
    if is_service_enabled q-l3; then
        # Configure and enable public bridge
        if is_neutron_ovs_base_plugin && [[ "$Q_USE_NAMESPACE" = "True" ]]; then
            local ext_gw_interface=$(_neutron_get_ext_gw_interface)
            local cidr_len=${FLOATING_RANGE#*/}
            sudo ip addr add $ext_gw_ip/$cidr_len dev $ext_gw_interface
            sudo ip link set $ext_gw_interface up
            ROUTER_GW_IP=`neutron port-list -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' -v subnet_id=$PUB_SUBNET_ID '$4 == subnet_id { print $8; }'`
            die_if_not_set $LINENO ROUTER_GW_IP "Failure retrieving ROUTER_GW_IP"
            sudo route add -net $FIXED_RANGE gw $ROUTER_GW_IP
        fi
        _neutron_set_router_id
    fi
}

# Configure neutron router for IPv6 public access
function _neutron_configure_router_v6 {
    neutron router-interface-add $ROUTER_ID $IPV6_SUBNET_ID
    # Create a public subnet on the external network
    local ipv6_id_and_ext_gw_ip=$(_neutron_create_public_subnet_v6 $EXT_NET_ID)
    local ipv6_ext_gw_ip=$(echo $ipv6_id_and_ext_gw_ip | get_field 2)
    local ipv6_pub_subnet_id=$(echo $ipv6_id_and_ext_gw_ip | get_field 5)

    # If the external network has not already been set as the default router
    # gateway when configuring an IPv4 public subnet, do so now
    if [[ "$IP_VERSION" == "6" ]]; then
        neutron router-gateway-set $ROUTER_ID $EXT_NET_ID
    fi

    # This logic is specific to using the l3-agent for layer 3
    if is_service_enabled q-l3; then
        local ipv6_router_gw_port
        # Ensure IPv6 forwarding is enabled on the host
        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        # Configure and enable public bridge
        if [[ "$IP_VERSION" = "6" ]]; then
            # Override global IPV6_ROUTER_GW_IP with the true value from neutron
            IPV6_ROUTER_GW_IP=`neutron port-list -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' -v subnet_id=$ipv6_pub_subnet_id '$4 == subnet_id { print $8; }'`
            die_if_not_set $LINENO IPV6_ROUTER_GW_IP "Failure retrieving IPV6_ROUTER_GW_IP"
            ipv6_router_gw_port=`neutron port-list -c id -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' -v subnet_id=$ipv6_pub_subnet_id '$4 == subnet_id { print $1; }' | awk -F ' | ' '{ print $2; }'`
            die_if_not_set $LINENO ipv6_router_gw_port "Failure retrieving ipv6_router_gw_port"
        else
            ipv6_router_gw_port=`neutron port-list -c id -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' -v subnet_id=$PUB_SUBNET_ID '$4 == subnet_id { print $1; }' | awk -F ' | ' '{ print $2; }'`
            die_if_not_set $LINENO ipv6_router_gw_port "Failure retrieving ipv6_router_gw_port"
        fi

        # The ovs_base_configure_l3_agent function flushes the public
        # bridge's ip addresses, so turn IPv6 support in the host off
        # and then on to recover the public bridge's link local address
        sudo sysctl -w net.ipv6.conf.${PUBLIC_BRIDGE}.disable_ipv6=1
        sudo sysctl -w net.ipv6.conf.${PUBLIC_BRIDGE}.disable_ipv6=0
        if is_neutron_ovs_base_plugin && [[ "$Q_USE_NAMESPACE" = "True" ]]; then
            local ext_gw_interface=$(_neutron_get_ext_gw_interface)
            local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}

            # Define router_ns based on whether DVR is enabled
            local router_ns=qrouter
            if [[ "$Q_DVR_MODE" == "dvr_snat" ]]; then
                router_ns=snat
            fi

            # Configure interface for public bridge
            sudo ip -6 addr add $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface

            # Wait until layer 3 agent has configured the gateway port on
            # the public bridge, then add gateway address to the interface
            # TODO (john-davidge) Remove once l3-agent supports dual-stack
            if [[ "$IP_VERSION" == "4+6" ]]; then
                if ! timeout $GATEWAY_TIMEOUT sh -c "until sudo ip netns exec $router_ns-$ROUTER_ID ip addr show qg-${ipv6_router_gw_port:0:11} | grep $ROUTER_GW_IP; do sleep 1; done"; then
                    die $LINENO "Timeout retrieving ROUTER_GW_IP"
                fi
                # Configure the gateway port with the public IPv6 adress
                sudo ip netns exec $router_ns-$ROUTER_ID ip -6 addr add $IPV6_ROUTER_GW_IP/$ipv6_cidr_len dev qg-${ipv6_router_gw_port:0:11}
                # Add a default IPv6 route to the neutron router as the
                # l3-agent does not add one in the dual-stack case
                sudo ip netns exec $router_ns-$ROUTER_ID ip -6 route replace default via $ipv6_ext_gw_ip dev qg-${ipv6_router_gw_port:0:11}
            fi
            sudo ip -6 route add $FIXED_RANGE_V6 via $IPV6_ROUTER_GW_IP dev $ext_gw_interface
        fi
        _neutron_set_router_id
    fi
}

# Explicitly set router id in l3 agent configuration
function _neutron_set_router_id {
    if [[ "$Q_USE_NAMESPACE" == "False" ]]; then
        iniset $Q_L3_CONF_FILE DEFAULT router_id $ROUTER_ID
    fi
}

# Get ext_gw_interface depending on value of Q_USE_PUBLIC_VETH
function _neutron_get_ext_gw_interface {
    if [[ "$Q_USE_PUBLIC_VETH" == "True" ]]; then
        echo $Q_PUBLIC_VETH_EX
    else
        # Disable in-band as we are going to use local port
        # to communicate with VMs
        sudo ovs-vsctl set Bridge $PUBLIC_BRIDGE \
            other_config:disable-in-band=true
        echo $PUBLIC_BRIDGE
    fi
}

# Functions for Neutron Exercises
#--------------------------------

function delete_probe {
    local from_net="$1"
    net_id=`_get_net_id $from_net`
    probe_id=`neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-list -c id -c network_id | grep $net_id | awk '{print $2}'`
    neutron-debug --os-tenant-name admin --os-username admin probe-delete $probe_id
}

function setup_neutron_debug {
    if [[ "$Q_USE_DEBUG_COMMAND" == "True" ]]; then
        public_net_id=`_get_net_id $PUBLIC_NETWORK_NAME`
        neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-create --device-owner compute $public_net_id
        private_net_id=`_get_net_id $PRIVATE_NETWORK_NAME`
        neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-create --device-owner compute $private_net_id
    fi
}

function teardown_neutron_debug {
    delete_probe $PUBLIC_NETWORK_NAME
    delete_probe $PRIVATE_NETWORK_NAME
}

function _get_net_id {
    neutron --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD net-list | grep $1 | awk '{print $2}'
}

function _get_probe_cmd_prefix {
    local from_net="$1"
    net_id=`_get_net_id $from_net`
    probe_id=`neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-list -c id -c network_id | grep $net_id | awk '{print $2}' | head -n 1`
    echo "$Q_RR_COMMAND ip netns exec qprobe-$probe_id"
}

function _ping_check_neutron {
    local from_net=$1
    local ip=$2
    local timeout_sec=$3
    local expected=${4:-"True"}
    local check_command=""
    probe_cmd=`_get_probe_cmd_prefix $from_net`
    if [[ "$expected" = "True" ]]; then
        check_command="while ! $probe_cmd ping -w 1 -c 1 $ip; do sleep 1; done"
    else
        check_command="while $probe_cmd ping -w 1 -c 1 $ip; do sleep 1; done"
    fi
    if ! timeout $timeout_sec sh -c "$check_command"; then
        if [[ "$expected" = "True" ]]; then
            die $LINENO "[Fail] Couldn't ping server"
        else
            die $LINENO "[Fail] Could ping server"
        fi
    fi
}

# ssh check
function _ssh_check_neutron {
    local from_net=$1
    local key_file=$2
    local ip=$3
    local user=$4
    local timeout_sec=$5
    local probe_cmd = ""
    probe_cmd=`_get_probe_cmd_prefix $from_net`
    if ! timeout $timeout_sec sh -c "while ! $probe_cmd ssh -o StrictHostKeyChecking=no -i $key_file ${user}@$ip echo success; do sleep 1; done"; then
        die $LINENO "server didn't become ssh-able!"
    fi
}

# Neutron 3rd party programs
#---------------------------

# please refer to ``lib/neutron_thirdparty/README.md`` for details
NEUTRON_THIRD_PARTIES=""
for f in $TOP_DIR/lib/neutron_thirdparty/*; do
    third_party=$(basename $f)
    if is_service_enabled $third_party; then
        source $TOP_DIR/lib/neutron_thirdparty/$third_party
        NEUTRON_THIRD_PARTIES="$NEUTRON_THIRD_PARTIES,$third_party"
    fi
done

function _neutron_third_party_do {
    for third_party in ${NEUTRON_THIRD_PARTIES//,/ }; do
        ${1}_${third_party}
    done
}

# configure_neutron_third_party() - Set config files, create data dirs, etc
function configure_neutron_third_party {
    _neutron_third_party_do configure
}

# init_neutron_third_party() - Initialize databases, etc.
function init_neutron_third_party {
    _neutron_third_party_do init
}

# install_neutron_third_party() - Collect source and prepare
function install_neutron_third_party {
    _neutron_third_party_do install
}

# start_neutron_third_party() - Start running processes, including screen
function start_neutron_third_party {
    _neutron_third_party_do start
}

# stop_neutron_third_party - Stop running processes (non-screen)
function stop_neutron_third_party {
    _neutron_third_party_do stop
}

# check_neutron_third_party_integration() - Check that third party integration is sane
function check_neutron_third_party_integration {
    _neutron_third_party_do check
}

function is_provider_network {
    if [ "$Q_USE_PROVIDER_NETWORKING" == "True" ] && [ "$Q_L3_ENABLED" == "False" ]; then
        return 0
    fi
    return 1
}


# Restore xtrace
$XTRACE

# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End: