Add devstack support

Add some scripts that devstack needs.

Change-Id: I8b9ad477ba0d2d9d581f90c67d5b46b32a4c66a3

.gitignore

@@ -7,6 +7,10 @@
 .stestr/
 *.DS_Store
 
+devstack/files/*.deb
+devstack/files/*.deb.*
+venv/
+
 dist
 build/*
 venus.egg-info/

MANIFEST.in

@@ -3,4 +3,6 @@ include ChangeLog
 exclude .gitignore
 exclude .gitreview
 
+recursive-include venus/db/sqlalchemy/migrate_repo *
+
 global-exclude *.pyc

devstack/fluentd-conf/filter/00-record_transformer.conf

@@ -0,0 +1,9 @@
+<filter *.log>
+    @type record_transformer
+    <record>
+        Hostname ${hostname}
+        Logger openstack.${tag_parts[4]}
+        programname ${tag_parts[5]}
+    </record>
+</filter>
+

devstack/fluentd-conf/filter/01-rewrite.conf.c

@@ -0,0 +1,4 @@
+<match *.log>
+    @type rewrite_tag_filter
+    capitalize_regex_backreference yes
+</match>

devstack/fluentd-conf/format/apache_access.conf

@@ -0,0 +1,10 @@
+<filter apache_access>
+  @type parser
+  reserve_data true
+  format grok
+  key_name message
+  grok_pattern \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:\d+|-)
+  time_key Timestamp
+  time_format %d/%b/%Y:%H:%M:%S %z
+  keep_time_key true
+</filter>

devstack/fluentd-conf/format/wsgi_access.conf

@@ -0,0 +1,10 @@
+<filter wsgi_access>
+  @type parser
+  reserve_data true
+  format grok
+  key_name message
+  grok_pattern %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:%{NUMBER:http_bytes}|-) (?:%{NUMBER:http_response_time_us}|-) %{QS:referrer} %{QS:agent}
+  time_key Timestamp
+  time_format %d/%b/%Y:%H:%M:%S %z
+  keep_time_key true
+</filter>

devstack/fluentd-conf/input/08-neutron.conf

@@ -0,0 +1,15 @@
+<source>
+  @type tail
+  path /var/log/kolla/neutron/*.log
+  pos_file /var/run/td-agent/devstack-openstack_neutron.pos
+  tag devstack
+  format multiline
+  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
+  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
+  time_key Timestamp
+  keep_time_key true
+  time_format %F %T.%L
+  ignore_repeated_permission_error true
+  enable_watch_timer false
+</source>
+

devstack/fluentd-conf/input/09-vitrage.conf

@@ -0,0 +1,15 @@
+<source>
+  @type tail
+  path /var/log/kolla/vitrage/*.log
+  pos_file /var/run/td-agent/devstack-openstack_vitrage.pos
+  tag devstack
+  format multiline
+  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
+  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
+  time_key Timestamp
+  keep_time_key true
+  time_format %F %T.%L
+  ignore_repeated_permission_error true
+  enable_watch_timer false
+</source>
+

devstack/fluentd-conf/input/10-keystone.conf

@@ -0,0 +1,15 @@
+<source>
+  @type tail
+  path /var/log/kolla/keystone/*.log
+  pos_file /var/run/td-agent/devstack-openstack_keystone.pos
+  tag devstack
+  format multiline
+  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
+  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
+  time_key Timestamp
+  keep_time_key true
+  time_format %F %T.%L
+  ignore_repeated_permission_error true
+  enable_watch_timer false
+</source>
+

devstack/fluentd-conf/input/11-cinder.conf

@@ -0,0 +1,15 @@
+<source>
+  @type tail
+  path /var/log/kolla/cinder/*.log
+  pos_file /var/run/td-agent/devstack-openstack_cinder.pos
+  tag devstack
+  format multiline
+  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
+  format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
+  time_key Timestamp
+  keep_time_key true
+  time_format %F %T.%L
+  ignore_repeated_permission_error true
+  enable_watch_timer false
+</source>
+

devstack/fluentd-conf/input/13-syslog.conf

@@ -0,0 +1,9 @@
+<source>
+  @type tail
+  path /var/log/bootstrap.log,/var/log/btmp,/var/log/syslog
+  pos_file /var/run/td-agent/devstack-syslog.pos
+  tag syslog.*
+  source_hostname_key Hostname
+  format /^(?<Payload>.*)$/
+  enable_watch_timer false
+</source>

devstack/fluentd-conf/output/00-local.conf.back

@@ -0,0 +1,18 @@
+<match syslog.**>
+  @type copy
+  <store>
+       type elasticsearch
+       host localhost
+       port 9200
+       logstash_format true
+       logstash_prefix slog
+       buffer_type memory
+       buffer_chunk_limit 8m
+       buffer_queue_limit 8
+       buffer_queue_full_action drop_oldest_chunk
+       flush_interval 30s
+       num_threads 4
+       reconnect_on_error true
+       resurrect_after 5s
+  </store>
+</match>

devstack/fluentd-conf/output/01-es.conf

@@ -0,0 +1,18 @@
+<match devstack>
+    @type copy
+    <store>
+       @type elasticsearch
+       host localhost
+       port 9200
+       logstash_format true
+       logstash_prefix flog
+       buffer_type memory
+       buffer_chunk_limit 8m
+       buffer_queue_limit 8
+       buffer_queue_full_action drop_oldest_chunk
+       flush_interval 30s
+       num_threads 4
+       reconnect_on_error true
+       resurrect_after 5s
+    </store>
+</match>

devstack/fluentd-conf/td-agent.conf

@@ -0,0 +1,5 @@
+@include input/*.conf
+@include filter/*.conf
+#@include format/*.conf
+@include output/*.conf
+

devstack/plugin.sh

@@ -0,0 +1,163 @@
+# plugin.sh - DevStack plugin.sh dispatch script venus
+
+# Support potential entry-points console scripts in VENV or not
+if [[ ${USE_VENV} == True ]]; then
+  PROJECT_VENV["venus"]=${VENUS_DIR}.venv
+  VENUS_BIN_DIR=${PROJECT_VENV["venus"]}/bin
+else
+  VENUS_BIN_DIR=$(get_python_exec_prefix)
+fi
+
+FILES=$VENUS_DIR/devstack/files
+
+function install_venus() {
+  setup_develop "$VENUS_DIR" openstack
+  install_fluentd
+  install_elastic_search
+}
+
+function init_venus() {
+  venus_create_accounts
+
+  recreate_database venus utf8
+  sudo $VENUS_BIN_DIR/venus-manage --config-file=${VENUS_CONF} db sync
+}
+
+function configure_venus() {
+  # Create venus conf directory
+  sudo install -d -o $STACK_USER -m 755 $VENUS_CONF_DIR
+
+  # Copy init conf file
+  sudo cp -R $VENUS_DIR/etc/venus/* $VENUS_CONF_DIR
+
+  iniset $VENUS_CONF keystone_authtoken memcached_servers localhost:11211
+  iniset $VENUS_CONF keystone_authtoken username venus
+  iniset $VENUS_CONF keystone_authtoken password "$ADMIN_PASSWORD"
+  iniset $VENUS_CONF keystone_authtoken auth_url "http://$HOST_IP/identity"
+
+  iniset $VENUS_CONF DEFAULT my_ip "$HOST_IP"
+  iniset $VENUS_CONF DEFAULT osapi_venus_listen_port 10010
+
+  iniset $VENUS_CONF database connection mysql+pymysql://root:"$DATABASE_PASSWORD"@localhost:3306/venus?charset=utf8
+
+  iniset $VENUS_CONF elasticsearch url http://localhost:9200
+}
+
+function start_venus() {
+  run_process venus-api "$VENUS_BIN_DIR/venus-api"
+}
+
+function install_elastic_search() {
+  echo_summary "install elastic search"
+  local FLUENTD_SERVICE="elasticsearch.service"
+  if [[ is_ubuntu ]]; then
+    install_package openjdk-8-jdk
+
+    ES_VERSION=${ES_VERSION:-5.6.16}
+    ES_DOWNLOAD_URL=${ES_DOWNLOAD_URL:-https://artifacts.elastic.co/downloads/elasticsearch}
+    ES_DOWNLOAD_FILE="elasticsearch-$ES_VERSION.deb"
+
+    if [[ ! -f $FILES/$ES_DOWNLOAD_FILE ]]; then
+      sudo wget --progress=dot:giga -t 2 -c $ES_DOWNLOAD_URL/$ES_DOWNLOAD_FILE -O $FILES/$ES_DOWNLOAD_FILE
+      if [[ $? -ne 0 ]]; then
+        die "$ES_DOWNLOAD_FILE could not be downloaded"
+      fi
+    fi
+
+    sudo dpkg -i $FILES/$ES_DOWNLOAD_FILE
+    $SYSTEMCTL daemon-reload
+    $SYSTEMCTL enable $FLUENTD_SERVICE
+    $SYSTEMCTL start $FLUENTD_SERVICE
+  else
+    exit_distro_not_supported "install elastic search"
+  fi
+}
+
+function install_fluentd() {
+  echo_summary "install fluentd"
+  if [[ is_ubuntu ]]; then
+    FLUENTD_VERSION=${FLUENTD_VERSION:-4.1.0-1_amd64}
+    FLUENTD_DOWNLOAD_URL=${FLUENTD_DOWNLOAD_URL:-http://packages.treasuredata.com.s3.amazonaws.com/4/ubuntu/bionic/pool/contrib/t/td-agent}
+    FLUENTD_DOWNLOAD_FILE="td-agent_$FLUENTD_VERSION.deb"
+
+    if [[ ! -f $FILES/$FLUENTD_DOWNLOAD_FILE ]]; then
+      sudo wget --progress=dot:giga -t 2 -c $FLUENTD_DOWNLOAD_URL/$FLUENTD_DOWNLOAD_FILE -O $FILES/$FLUENTD_DOWNLOAD_FILE
+      if [[ $? -ne 0 ]]; then
+        die "$FLUENTD_DOWNLOAD_FILE could not be downloaded"
+      fi
+    fi
+
+    sudo dpkg -i $FILES/$FLUENTD_DOWNLOAD_FILE
+  else
+    exit_distro_not_supported "install fluentd"
+  fi
+
+  # Create log dir
+  VENUS_LOG_DIR="/var/log/kolla"
+  sudo install -d -o $STACK_USER -m 777 $VENUS_LOG_DIR
+
+  # Copy fluentd conf
+  sudo cp -R $VENUS_DIR/devstack/fluentd-conf/* /etc/td-agent
+  $SYSTEMCTL restart td-agent
+}
+
+function venus_create_accounts() {
+  create_service_user "venus"
+}
+
+function uninstall_elastic_search() {
+  local ELASTIC_SEARCH_SERVICE="elasticsearch.service"
+  $SYSTEMCTL stop $ELASTIC_SEARCH_SERVICE
+  $SYSTEMCTL disable $ELASTIC_SEARCH_SERVICE
+  $SYSTEMCTL daemon-reload
+  sudo dpkg -r elasticsearch
+}
+
+function uninstall_fluentd() {
+  local FLUENTD_SERVICE="td-agent.service"
+  $SYSTEMCTL stop $FLUENTD_SERVICE
+  $SYSTEMCTL disable $FLUENTD_SERVICE
+  $SYSTEMCTL daemon-reload
+  sudo dpkg -r td-agent
+}
+
+# check for service enabled
+if is_service_enabled venus-api; then
+
+  if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
+    # Set up system services
+    echo_summary "Configuring system services venus"
+    echo_summary "Welcome to Venus!"
+
+  elif [[ "$1" == "stack" && "$2" == "install" ]]; then
+    # Perform installation of service source
+    echo_summary "Installing venus"
+    install_venus
+
+  elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
+    # Configure after the other layer 1 and 2 services have been configured
+    echo_summary "Configuring venus"
+    configure_venus
+
+  elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
+    # Initialize and start the venus service
+    echo_summary "Initializing venus"
+    init_venus
+    start_venus
+  fi
+
+  if [[ "$1" == "unstack" ]]; then
+    # Shut down venus services
+    # no-op
+    :
+  fi
+
+  if [[ "$1" == "clean" ]]; then
+    # Remove state and transient data
+    # Remember clean.sh first calls unstack.sh
+    # no-op
+    uninstall_elastic_search
+    uninstall_fluentd
+    sudo rm -rf $VENUS_CONF_DIR
+  fi
+fi

devstack/settings

@@ -0,0 +1,9 @@
+# settings file for venus
+
+# API Service
+enable_service venus-api
+
+# Default directories
+VENUS_DIR=$DEST/venus
+VENUS_CONF_DIR=/etc/venus
+VENUS_CONF=$VENUS_CONF_DIR/venus.conf

etc/venus/venus.conf

@@ -1,37 +1,27 @@
 [keystone_authtoken]
-memcached_servers = 100.2.30.241:11211,100.2.30.242:11211,100.2.30.243:11211signing_dir = /var/cache/venus
-signing_dir = /var/cache/venus
-cafile = /opt/stack/data/ca-bundle.pem
-project_domain_name = default
+project_domain_name = Default
 project_name = service
-user_domain_name = default
-password = dTa74mdF29CyGLQvH8RCKAhFPlRd1zHtp2Ai4NGw
+user_domain_name = Default
+password = secret
 username = venus
-auth_uri = http://100.2.28.240:5000
-auth_url = http://100.2.28.240:35357
-project_domain_id = default
-user_domain_id = default
+auth_url = http://0.0.0.0/identity
 auth_type = password
 
 [DEFAULT]
-transport_url = rabbit://openstack:R8axM8sde8Dq5tV1PcDHmDRPLsA9fBLpXrGQccfE@100.2.30.243:5672
-my_ip = 100.2.30.243
+transport_url = rabbit://openstack:secret@0.0.0.0:5672
+my_ip = 0.0.0.0
 periodic_interval = 60
 rootwrap_config = /etc/venus/rootwrap.conf
 api_paste_config = /etc/venus/api-paste.ini
-log_dir = /var/log/kolla/venus/
+use_syslog = False
 debug = True
-use_stderr = True
 auth_strategy = keystone
 os_region_name = RegionOne
-osapi_venus_listen = 100.2.30.243
-osapi_venus_listen_port = 8686
+osapi_venus_listen_port = 10010
 osapi_venus_workers = 1
 
 [database]
-connection = mysql+pymysql://root:Irpzw6tic9ezyUEh4c0JnT0kK7U1oKqbRPRIfkwW@100.2.28.72:3306/venus?charset=utf8
+connection = mysql+pymysql://root:secret@localhost:3306/venus?charset=utf8
 
 [elasticsearch]
-url = http://100.2.28.30:9200
-username = admin
-password = DlR7Y4vcPPbwbOCHYO8f8zG9VtwnLrd1t5R1A3B9
+url = http://localhost:9200

setup.cfg

@@ -24,8 +24,6 @@ packages =
     venus
 
 [entry_points]
-venus.database.migration_backend =
-    sqlalchemy = venus.db.sqlalchemy.migration
-
-wsgi_scripts =
-    venus-api-wsgi = venus.cmd.api:main
+console_scripts =
+    venus-manage = venus.cmd.manage:main
+    venus-api = venus.cmd.api:main

venus/cmd/api.py

@@ -15,7 +15,6 @@
 """Starter script for Venus OS API."""
 
 import eventlet
-import os
 import sys
 
 from oslo_log import log as logging
@@ -36,10 +35,10 @@ def main():
     objects.register_all()
     CONF(sys.argv[1:], project='venus',
          version=version.version_string())
-    logdir = CONF.log_dir
-    is_exits = os.path.exists(logdir)
-    if not is_exits:
-        os.makedirs(logdir)
+    # logdir = CONF.log_dir
+    # is_exits = os.path.exists(logdir)
+    # if not is_exits:
+    #     os.makedirs(logdir)
     logging.setup(CONF, "venus")
     utils.monkey_patch()
 

venus/cmd/manage.py

@@ -345,10 +345,10 @@ def main():
     try:
         CONF(sys.argv[1:], project='venus',
              version=version.version_string())
-        logdir = CONF.log_dir
-        is_exits = os.path.exists(logdir)
-        if not is_exits:
-            os.makedirs(logdir)
+        # logdir = CONF.log_dir
+        # is_exits = os.path.exists(logdir)
+        # if not is_exits:
+        #     os.makedirs(logdir)
         logging.setup(CONF, "venus")
     except cfg.ConfigDirNotFoundError as details:
         print(_("Invalid directory: %s") % details)

venus/conf/api.py

@@ -23,7 +23,7 @@ service_opts = [
                default=60,
                help='Range, in seconds, to randomly delay when starting the'
                     ' periodic task scheduler to reduce stampeding.'
-                    ' (Disable by setting to 0)'),
+                    ' (Disable by settings to 0)'),
     cfg.StrOpt('osapi_venus_listen',
                default="0.0.0.0",
                help='IP address on which OpenStack Venus API listens'),

venus/conf/common.py

@@ -134,7 +134,8 @@ global_opts = [
                 default=True,
                 help='If False, closes the client socket connection '
                      'explicitly. Setting it to True to maintain backward '
-                     'compatibility. Recommended setting is set it to False.'),
+                     'compatibility. Recommended settings is set it '
+                     'to False.'),
     cfg.BoolOpt('fatal_exception_format_errors',
                 default=False,
                 help='Make exception message format errors fatal.'),

venus/db/migration.py

@@ -17,42 +17,41 @@
 import os
 import threading
 
-from oslo_db import options
-from stevedore import driver
-
-from venus.conf import CONF
-from venus.db.sqlalchemy import api as db_api
+from oslo_config import cfg
+from oslo_db import options as db_options
+from oslo_db.sqlalchemy.migration import db_sync as sync
+from oslo_db.sqlalchemy import session as db_session
 
 INIT_VERSION = 000
 
 _IMPL = None
 _LOCK = threading.Lock()
 
-options.set_defaults(CONF)
+db_options.set_defaults(cfg.CONF)
 
 MIGRATE_REPO_PATH = os.path.join(
     os.path.abspath(os.path.dirname(__file__)),
-    'sqlalchemy',
-    'migrate_repo',
+    'sqlalchemy/migrate_repo',
 )
 
 
-def get_backend():
+def get_engine():
+    print(cfg.CONF)
     global _IMPL
     if _IMPL is None:
         with _LOCK:
             if _IMPL is None:
-                _IMPL = driver.DriverManager(
-                    "venus.database.migration_backend",
-                    CONF.database.backend).driver
-    return _IMPL
+                _IMPL = db_session.EngineFacade(
+                    cfg.CONF.database.connection,
+                    **dict(cfg.CONF.database)
+                )
+    return _IMPL.get_engine()
 
 
 def db_sync(version=None, init_version=INIT_VERSION, engine=None):
     """Migrate the database to `version` or the most recent version."""
-    if engine is None:
-        engine = db_api.get_engine()
-    return get_backend().db_sync(engine=engine,
+    engine = get_engine()
+    return sync(engine=engine,
                 abs_path=MIGRATE_REPO_PATH,
                 version=version,
                 init_version=init_version)

zzh/readme.txt

@@ -1 +0,0 @@
-My fist commit.