airship/sip
Code Issues Proposed changes

Worker Load Balancer

Browse Source 
Spin up a new load balancer for services exposed on the
sub-cluster(ControlPlane and Worker)

TODO: Support node port range
Map each port to same port on sub-cluster (instead of 6443)
Optimize haproxy config for generic workload services

Change-Id: I59125d7af06886fe128d068f657f0d9f1be7e926
This commit is contained in:
francisy 2021-03-16 18:16:56 -04:00
parent 8c7d108831
commit 03085759e8
11 changed files with 498 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
config/crd/bases/airship.airshipit.org_sipclusters.yaml
View File

@ -178,10 +178,36 @@ spec:
- nodeSSHPrivateKeys
type: object
type: array
loadBalancer:
loadBalancerControlPlane:
description: LoadBalancer defines the sub-cluster load balancer
services.
items:
description: LoadBalancerServiceControlPlane is an infrastructure
service type that represents the sub-cluster load balancer service.
properties:
clusterIP:
type: string
image:
type: string
nodeInterfaceId:
type: string
nodeLabels:
additionalProperties:
type: string
type: object
nodePort:
type: integer
required:
- image
- nodePort
type: object
type: array
loadBalancerWorker:
description: ' LoadBalancer defines the sub-cluster load balancer
services.'
items:
description: LoadBalancerServiceWorker is an infrastructure service
type that represents the sub-cluster load balancer service.
properties:
clusterIP:
type: string

2
config/rbac/sipcluster_scheduler_role.yaml
View File

@ -17,8 +17,10 @@ rules:
- update
- apiGroups:
- ""
- apps
resources:
- secrets
- deployments
verbs:
- get
- list

40
config/samples/airship_v1beta1_sipcluster.yaml
View File

@ -8,19 +8,19 @@ metadata:
spec:
nodes:
ControlPlane:
labelSelector:
vino.airshipit.org/flavor: control-plane
labelSelector:
vino.airshipit.org/flavor: control-plane
topologyKey: vino.airshipit.org/rack
count:
active: 1
standby: 1
count:
active: 1
standby: 1
Worker:
labelSelector:
vino.airshipit.org/flavor: worker
topologyKey: vino.airshipit.org/host
count:
active: 1
standby: 1 # Slew for upgrades
labelSelector:
vino.airshipit.org/flavor: worker
topologyKey: vino.airshipit.org/host
count:
active: 1
standby: 1 # Slew for upgrades
services:
# NOTE: The auth service has not yet been implemented.
# auth:
@ -36,7 +36,7 @@ spec:
# NOTE: nodeLabels not yet implemented.
# nodeLabels:
# kubernetes.io/os: linux
nodePort: 30001
nodePort: 30000
nodeInterfaceId: oam-ipv4
# NOTE: clusterIP has not yet been implemented.
# clusterIP: 1.2.3.4 # IP of the base cluster VIP
@ -46,13 +46,21 @@ spec:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCyaozS8kZRw2a1d0O4YXhxtJlDPThqIZilGCsXLbukIFOyMUmMTwQAtwWp5epwU1+5ponC2uBENB6xCCj3cl5Rd43d2/B6HxyAPQGKo6/zKYGAKW2nzYDxSWMl6NUSsiJAyXUA7ZlNZQe0m8PmaferlkQyLLZo3NJpizz6U6ZCtxvj43vEl7NYWnLUEIzGP9zMqltIGnD4vYrU9keVKKXSsp+DkApnbrDapeigeGATCammy2xRrUQDuOvGHsfnQbXr2j0onpTIh0PiLrXLQAPDg8UJRgVB+ThX+neI3rQ320djzRABckNeE6e4Kkwzn+QdZsmA2SDvM9IU7boK1jVQlgUPp7zF5q3hbb8Rx7AadyTarBayUkCgNlrMqth+tmTMWttMqCPxJRGnhhvesAHIl55a28Kzz/2Oqa3J9zwzbyDIwlEXho0eAq3YXEPeBhl34k+7gOt/5Zdbh+yacFoxDh0LrshQgboAijcVVaXPeN0LsHEiVvYIzugwIvCkoFMPWoPj/kEGzPY6FCkVneDA7VoLTCoG8dlrN08Lf05/BGC7Wllm66pTNZC/cKXP+cjpQn1iEuiuPxnPldlMHx9sx2y/BRoft6oT/GzqkNy1NTY/xI+MfmxXnF5kwSbcTbzZQ9fZ8xjh/vmpPBgDNrxOEAT4N6OG7GQIhb9HEhXQCQ== example-key
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwpOyZjZ4gB0OTvmofH3llh6cBCWaEiEmHZWSkDXr8Bih6HcXVOtYMcFi/ZnUVGUBPw3ATNQBZUaVCYKeF+nDfKTJ9hmnlsyHxV2LeMsVg1o15Pb6f+QJuavEqtE6HI7mHyId4Z1quVTJXDWDW8OZEG7M3VktauqAn/e9UJvlL0bGmTFD1XkNcbRsWMRWkQgt2ozqlgrpPtvrg2/+bNucxX++VUjnsn+fGgAT07kbnrZwppGnAfjbYthxhv7GeSD0+Z0Lf1kiKy/bhUqXsZIuexOfF0YrRyUH1KBl8GCX2OLBYvXHyusByqsrOPiROqRdjX5PsK6HSAS0lk0niTt1p example-key-2
nodeSSHPrivateKeys: ssh-private-keys
loadBalancer:
loadBalancerControlPlane:
- image: haproxy:2.3.2
# NOTE: nodeLabels not yet implemented.
# nodeLabels:
# kubernetes.io/os: linux
nodePort: 30000
# kubernetes.io/os
nodePort: 30001
nodeInterfaceId: oam-ipv4
# NOTE: clusterIP has not yet been implemented.
# clusterIP: 1.2.3.4 # IP of the base cluster VIP
loadBalancerWorker:
- image: haproxy:2.3.2
# NOTE: nodeLabels not yet implemented.
# nodeLabels:
# kubernetes.io/os
nodePort: 30002
nodeInterfaceId: oam-ipv4
# NOTE: clusterIP has not yet been implemented.
# clusterIP: 1.2.3.4 # IP of the base cluster VIP

93
docs/api/sipcluster.md
View File

@ -116,6 +116,78 @@ directory, and then configured as identity files in the SSH config file of the d
</table>
</div>
</div>
<h3 id="airship.airshipit.org/v1.LoadBalancerServiceControlPlane">LoadBalancerServiceControlPlane
</h3>
<p>
(<em>Appears on:</em>
<a href="#airship.airshipit.org/v1.SIPClusterServices">SIPClusterServices</a>)
</p>
<p>LoadBalancerServiceControlPlane is an infrastructure service type that represents the sub-cluster load balancer service.</p>
<div class="md-typeset__scrollwrap">
<div class="md-typeset__table">
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>SIPClusterService</code><br>
<em>
<a href="#airship.airshipit.org/v1.SIPClusterService">
SIPClusterService
</a>
</em>
</td>
<td>
<p>
(Members of <code>SIPClusterService</code> are embedded into this type.)
</p>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<h3 id="airship.airshipit.org/v1.LoadBalancerServiceWorker">LoadBalancerServiceWorker
</h3>
<p>
(<em>Appears on:</em>
<a href="#airship.airshipit.org/v1.SIPClusterServices">SIPClusterServices</a>)
</p>
<p>LoadBalancerServiceWorker is an infrastructure service type that represents the sub-cluster load balancer service.</p>
<div class="md-typeset__scrollwrap">
<div class="md-typeset__table">
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>SIPClusterService</code><br>
<em>
<a href="#airship.airshipit.org/v1.SIPClusterService">
SIPClusterService
</a>
</em>
</td>
<td>
<p>
(Members of <code>SIPClusterService</code> are embedded into this type.)
</p>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<h3 id="airship.airshipit.org/v1.NodeCount">NodeCount
</h3>
<p>
@ -317,6 +389,8 @@ SIPClusterStatus
<p>
(<em>Appears on:</em>
<a href="#airship.airshipit.org/v1.JumpHostService">JumpHostService</a>,
<a href="#airship.airshipit.org/v1.LoadBalancerServiceControlPlane">LoadBalancerServiceControlPlane</a>,
<a href="#airship.airshipit.org/v1.LoadBalancerServiceWorker">LoadBalancerServiceWorker</a>,
<a href="#airship.airshipit.org/v1.SIPClusterServices">SIPClusterServices</a>)
</p>
<div class="md-typeset__scrollwrap">
@ -402,10 +476,23 @@ string
<tbody>
<tr>
<td>
<code>loadBalancer</code><br>
<code>loadBalancerControlPlane</code><br>
<em>
<a href="#airship.airshipit.org/v1.SIPClusterService">
[]SIPClusterService
<a href="#airship.airshipit.org/v1.LoadBalancerServiceControlPlane">
[]LoadBalancerServiceControlPlane
</a>
</em>
</td>
<td>
<p>LoadBalancer defines the sub-cluster load balancer services.</p>
</td>
</tr>
<tr>
<td>
<code>loadBalancerWorker</code><br>
<em>
<a href="#airship.airshipit.org/v1.LoadBalancerServiceWorker">
[]LoadBalancerServiceWorker
</a>
</em>
</td>

24
pkg/api/v1/sipcluster_types.go
View File

@ -53,7 +53,10 @@ type SIPClusterSpec struct {
// SIPClusterServices defines the services that are deployed when a SIPCluster is provisioned.
type SIPClusterServices struct {
// LoadBalancer defines the sub-cluster load balancer services.
LoadBalancer []SIPClusterService `json:"loadBalancer,omitempty"`
LoadBalancerControlPlane []LoadBalancerServiceControlPlane `json:"loadBalancerControlPlane,omitempty"`
// LoadBalancer defines the sub-cluster load balancer services.
LoadBalancerWorker []LoadBalancerServiceWorker `json:"loadBalancerWorker,omitempty"`
// Auth defines the sub-cluster authentication services.
Auth []SIPClusterService `json:"auth,omitempty"`
// JumpHost defines the sub-cluster jump host services.
@ -62,8 +65,11 @@ type SIPClusterServices struct {
func (s SIPClusterServices) GetAll() []SIPClusterService {
all := []SIPClusterService{}
for _, s := range s.LoadBalancer {
all = append(all, s)
for _, s := range s.LoadBalancerControlPlane {
all = append(all, s.SIPClusterService)
}
for _, s := range s.LoadBalancerWorker {
all = append(all, s.SIPClusterService)
}
for _, s := range s.Auth {
all = append(all, s)
@ -86,6 +92,18 @@ type JumpHostService struct {
NodeSSHPrivateKeys string `json:"nodeSSHPrivateKeys"`
}
/*
LoadBalancerServiceControlPlane is an infrastructure service type that represents the sub-cluster load balancer service.
*/
type LoadBalancerServiceControlPlane struct {
SIPClusterService `json:",inline"`
}
// LoadBalancerServiceWorker is an infrastructure service type that represents the sub-cluster load balancer service.
type LoadBalancerServiceWorker struct {
SIPClusterService `json:",inline"`
}
// SIPClusterStatus defines the observed state of SIPCluster
type SIPClusterStatus struct {
Conditions []metav1.Condition `json:"conditions,omitempty"`

45
pkg/api/v1/zz_generated.deepcopy.go
View File

@ -66,6 +66,38 @@ func (in *JumpHostService) DeepCopy() *JumpHostService {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancerServiceControlPlane) DeepCopyInto(out *LoadBalancerServiceControlPlane) {
*out = *in
in.SIPClusterService.DeepCopyInto(&out.SIPClusterService)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerServiceControlPlane.
func (in *LoadBalancerServiceControlPlane) DeepCopy() *LoadBalancerServiceControlPlane {
if in == nil {
return nil
}
out := new(LoadBalancerServiceControlPlane)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancerServiceWorker) DeepCopyInto(out *LoadBalancerServiceWorker) {
*out = *in
in.SIPClusterService.DeepCopyInto(&out.SIPClusterService)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerServiceWorker.
func (in *LoadBalancerServiceWorker) DeepCopy() *LoadBalancerServiceWorker {
if in == nil {
return nil
}
out := new(LoadBalancerServiceWorker)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeCount) DeepCopyInto(out *NodeCount) {
*out = *in
@ -191,9 +223,16 @@ func (in *SIPClusterService) DeepCopy() *SIPClusterService {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SIPClusterServices) DeepCopyInto(out *SIPClusterServices) {
*out = *in
if in.LoadBalancer != nil {
in, out := &in.LoadBalancer, &out.LoadBalancer
*out = make([]SIPClusterService, len(*in))
if in.LoadBalancerControlPlane != nil {
in, out := &in.LoadBalancerControlPlane, &out.LoadBalancerControlPlane
*out = make([]LoadBalancerServiceControlPlane, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.LoadBalancerWorker != nil {
in, out := &in.LoadBalancerWorker, &out.LoadBalancerWorker
*out = make([]LoadBalancerServiceWorker, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}

156
pkg/bmh/bmh_test.go
View File

@ -128,14 +128,28 @@ var _ = Describe("MachineList", func() {
sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3)
sipCluster.Spec.Services = airshipv1.SIPClusterServices{
LoadBalancer: []airshipv1.SIPClusterService{
LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{
{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30001,
NodeInterface: "oam-ipv4",
},
},
},
LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30002,
NodeInterface: "oam-ipv4",
},
NodePort: 30000,
NodeInterface: "oam-ipv4",
},
},
}
@ -179,14 +193,28 @@ var _ = Describe("MachineList", func() {
sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3)
sipCluster.Spec.Services = airshipv1.SIPClusterServices{
LoadBalancer: []airshipv1.SIPClusterService{
LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{
{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30001,
NodeInterface: "oam-ipv4",
},
},
},
LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30002,
NodeInterface: "oam-ipv4",
},
NodePort: 30000,
NodeInterface: "oam-ipv4",
},
},
}
@ -228,14 +256,28 @@ var _ = Describe("MachineList", func() {
sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3)
sipCluster.Spec.Services = airshipv1.SIPClusterServices{
LoadBalancer: []airshipv1.SIPClusterService{
LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{
{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30001,
NodeInterface: "oam-ipv4",
},
},
},
LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30002,
NodeInterface: "oam-ipv4",
},
NodePort: 30000,
NodeInterface: "oam-ipv4",
},
},
}
@ -281,14 +323,28 @@ var _ = Describe("MachineList", func() {
sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3)
sipCluster.Spec.Services = airshipv1.SIPClusterServices{
LoadBalancer: []airshipv1.SIPClusterService{
LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{
{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30001,
NodeInterface: "oam-ipv4",
},
},
},
LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30002,
NodeInterface: "oam-ip4",
},
NodePort: 30000,
NodeInterface: "oam-ipv4",
},
},
}
@ -328,14 +384,28 @@ var _ = Describe("MachineList", func() {
sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3)
sipCluster.Spec.Services = airshipv1.SIPClusterServices{
LoadBalancer: []airshipv1.SIPClusterService{
LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{
{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30001,
NodeInterface: "oam-ipv4",
},
},
},
LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30002,
NodeInterface: "oam-ipv4",
},
NodePort: 30000,
NodeInterface: "oam-ipv4",
},
},
}
@ -374,14 +444,28 @@ var _ = Describe("MachineList", func() {
sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3)
sipCluster.Spec.Services = airshipv1.SIPClusterServices{
LoadBalancer: []airshipv1.SIPClusterService{
LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{
{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30001,
NodeInterface: "oam-ipv4",
},
},
},
LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "haproxy:latest",
NodeLabels: map[string]string{
"test": "true",
},
NodePort: 30002,
NodeInterface: "oam-ipv4",
},
NodePort: 30000,
NodeInterface: "oam-ipv4",
},
},
}

109
pkg/services/loadbalancer.go
View File

@ -16,6 +16,7 @@ package services
import (
"bytes"
"strings"
"html/template"
airshipv1 "sipcluster/pkg/api/v1"
@ -44,7 +45,7 @@ func (lb loadBalancer) Deploy() error {
lb.config.Image = DefaultBalancerImage
}
instance := LoadBalancerServiceName + "-" + lb.sipName.Name
instance := LoadBalancerServiceName + "-" + strings.ToLower(string(lb.bmhRole)) + "-" + lb.sipName.Name
labels := map[string]string{
// See https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels
"app.kubernetes.io/part-of": "sip",
@ -149,7 +150,7 @@ func (lb loadBalancer) generateSecret(instance string) (*corev1.Secret, error) {
Backends: make([]backend, 0),
}
for _, machine := range lb.machines.Machines {
if machine.BMHRole == airshipv1.RoleControlPlane {
if machine.BMHRole == lb.bmhRole {
name := machine.BMH.Name
namespace := machine.BMH.Namespace
ip, exists := machine.Data.IPOnInterface[lb.config.NodeInterface]
@ -163,7 +164,7 @@ func (lb loadBalancer) generateSecret(instance string) (*corev1.Secret, error) {
p.Backends = append(p.Backends, backend{IP: ip, Name: machine.BMH.Name, Port: 6443})
}
}
secretData, err := generateTemplate(p)
secretData, err := lb.generateTemplate(p)
if err != nil {
return nil, err
}
@ -216,22 +217,59 @@ type loadBalancer struct {
logger logr.Logger
config airshipv1.SIPClusterService
machines *bmh.MachineList
bmhRole airshipv1.BMHRole
template string
}
func newLB(name, namespace string,
type loadBalancerControlPlane struct {
loadBalancer
config airshipv1.LoadBalancerServiceControlPlane
}
type loadBalancerWorker struct {
loadBalancer
config airshipv1.LoadBalancerServiceWorker
}
func newLBControlPlane(name, namespace string,
logger logr.Logger,
config airshipv1.SIPClusterService,
config airshipv1.LoadBalancerServiceControlPlane,
machines *bmh.MachineList,
client client.Client) loadBalancer {
return loadBalancer{
client client.Client) loadBalancerControlPlane {
return loadBalancerControlPlane{loadBalancer{
sipName: types.NamespacedName{
Name: name,
Namespace: namespace,
},
logger: logger,
config: config,
config: config.SIPClusterService,
machines: machines,
client: client,
bmhRole: airshipv1.RoleControlPlane,
template: templateControlPlane,
},
config,
}
}
func newLBWorker(name, namespace string,
logger logr.Logger,
config airshipv1.LoadBalancerServiceWorker,
machines *bmh.MachineList,
client client.Client) loadBalancerWorker {
return loadBalancerWorker{loadBalancer{
sipName: types.NamespacedName{
Name: name,
Namespace: namespace,
},
logger: logger,
config: config.SIPClusterService,
machines: machines,
client: client,
bmhRole: airshipv1.RoleWorker,
template: templateWorker,
},
config,
}
}
@ -240,8 +278,8 @@ func (lb loadBalancer) Finalize() error {
return nil
}
func generateTemplate(p proxy) ([]byte, error) {
tmpl, err := template.New("haproxy-config").Parse(defaultTemplate)
func (lb loadBalancer) generateTemplate(p proxy) ([]byte, error) {
tmpl, err := template.New("haproxy-config").Parse(lb.template)
if err != nil {
return nil, err
}
@ -255,7 +293,7 @@ func generateTemplate(p proxy) ([]byte, error) {
return rendered, nil
}
var defaultTemplate = `global
var templateControlPlane = `global
log stdout format raw local0 notice
daemon
@ -305,3 +343,52 @@ backend kube-apiservers
{{- $backEnd := . }}
server {{ $backEnd.Name }} {{ $backEnd.IP }}:{{ $backEnd.Port }}
{{ end -}}`
// TODO Update this template to work for workload services, as it currently references api server(control plane)
var templateWorker = `global
log stdout format raw local0 notice
daemon
defaults
mode http
log global
option httplog
option dontlognull
retries 1
# Configures the timeout for a connection request to be left pending in a queue
# (connection requests are queued once the maximum number of connections is reached).
timeout queue 30s
# Configures the timeout for a connection to a backend server to be established.
timeout connect 30s
# Configures the timeout for inactivity during periods when we would expect
# the client to be speaking. For usability of 'kubectl exec', the timeout should
# be long enough to cover inactivity due to idleness of interactive sessions.
timeout client 600s
# Configures the timeout for inactivity during periods when we would expect
# the server to be speaking. For usability of 'kubectl log -f', the timeout should
# be long enough to cover inactivity due to the lack of new logs.
timeout server 600s
#---------------------------------------------------------------------
# apiserver frontend which proxys to the masters
#---------------------------------------------------------------------
frontend apiserver
bind *:{{ .FrontPort }}
mode tcp
option tcplog
default_backend kube-apiservers
#---------------------------------------------------------------------
# round robin balancing for apiserver
#---------------------------------------------------------------------
backend kube-apiservers
mode tcp
balance roundrobin
option httpchk GET /readyz
http-check expect status 200
option log-health-checks
# Observed apiserver returns 500 for around 10s when 2nd cp node joins.
# downinter 2s makes it check more frequently to recover from that state sooner.
# Also changing fall to 4 so that it takes longer (4 failures) for it to take down a backend.
default-server check check-ssl verify none inter 5s downinter 2s fall 4 on-marked-down shutdown-sessions
{{- range .Backends }}
{{- $backEnd := . }}
server {{ $backEnd.Name }} {{ $backEnd.IP }}:{{ $backEnd.Port }}
{{ end -}}`

57
pkg/services/services_test.go
View File

@ -3,6 +3,7 @@ package services_test
import (
"context"
"encoding/json"
"strings"
airshipv1 "sipcluster/pkg/api/v1"
@ -106,7 +107,7 @@ var _ = Describe("Service Set", func() {
set := services.NewServiceSet(logger, *sipCluster, machineList, k8sClient)
serviceList, err := set.ServiceList()
Expect(serviceList).To(HaveLen(2))
Expect(serviceList).To(HaveLen(3))
Expect(err).To(Succeed())
for _, svc := range serviceList {
err := svc.Deploy()
@ -121,7 +122,8 @@ var _ = Describe("Service Set", func() {
It("Does not deploy a Jump Host when an invalid SSH key is provided", func() {
sip, _ := testutil.CreateSIPCluster("default", "default", 1, 1)
sip.Spec.Services.Auth = []airshipv1.SIPClusterService{}
sip.Spec.Services.LoadBalancer = []airshipv1.SIPClusterService{}
sip.Spec.Services.LoadBalancerControlPlane = []airshipv1.LoadBalancerServiceControlPlane{}
sip.Spec.Services.LoadBalancerWorker = []airshipv1.LoadBalancerServiceWorker{}
sip.Spec.Services.JumpHost[0].SSHAuthorizedKeys = []string{
"sshrsaAAAAAAAAAAAAAAAAAAAAAinvalidkey",
}
@ -141,29 +143,62 @@ var _ = Describe("Service Set", func() {
})
func testDeployment(sip *airshipv1.SIPCluster, machineList bmh.MachineList) error {
loadBalancerDeployment := &appsv1.Deployment{}
loadBalancerControlPlaneDeployment := &appsv1.Deployment{}
err := k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.LoadBalancerServiceName + "-" + sip.GetName(),
}, loadBalancerDeployment)
Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleControlPlane)) + "-" +
sip.GetName(),
}, loadBalancerControlPlaneDeployment)
if err != nil {
return err
}
loadBalancerSecret := &corev1.Secret{}
loadBalancerWorkerDeployment := &appsv1.Deployment{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.LoadBalancerServiceName + "-" + sip.GetName(),
}, loadBalancerSecret)
Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleWorker)) + "-" +
sip.GetName(),
}, loadBalancerWorkerDeployment)
if err != nil {
return err
}
loadBalancerService := &corev1.Service{}
loadBalancerControlPlaneSecret := &corev1.Secret{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.LoadBalancerServiceName + "-" + sip.GetName(),
}, loadBalancerService)
Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleControlPlane)) + "-" +
sip.GetName(),
}, loadBalancerControlPlaneSecret)
if err != nil {
return err
}
loadBalancerWorkerSecret := &corev1.Secret{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleWorker)) + "-" +
sip.GetName(),
}, loadBalancerWorkerSecret)
if err != nil {
return err
}
loadBalancerControlPlaneService := &corev1.Service{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleControlPlane)) + "-" +
sip.GetName(),
}, loadBalancerControlPlaneService)
if err != nil {
return err
}
loadBalancerWorkerService := &corev1.Service{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleWorker)) + "-" +
sip.GetName(),
}, loadBalancerWorkerService)
if err != nil {
return err
}

13
pkg/services/set.go
View File

@ -64,9 +64,18 @@ func (ss ServiceSet) Finalize() error {
func (ss ServiceSet) ServiceList() ([]InfraService, error) {
serviceList := []InfraService{}
services := ss.sip.Spec.Services
for _, svc := range services.LoadBalancer {
for _, svc := range services.LoadBalancerControlPlane {
serviceList = append(serviceList,
newLB(ss.sip.GetName(),
newLBControlPlane(ss.sip.GetName(),
ss.sip.GetNamespace(),
ss.logger,
svc,
ss.machines,
ss.client))
}
for _, svc := range services.LoadBalancerWorker {
serviceList = append(serviceList,
newLBWorker(ss.sip.GetName(),
ss.sip.GetNamespace(),
ss.logger,
svc,

24
testutil/testutil.go
View File

@ -261,17 +261,11 @@ func CreateSIPCluster(name string, namespace string, controlPlanes int, workers
},
},
Services: airshipv1.SIPClusterServices{
LoadBalancer: []airshipv1.SIPClusterService{
{
NodeInterface: "eno3",
NodePort: 30000,
},
},
JumpHost: []airshipv1.JumpHostService{
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "quay.io/airshipit/jump-host",
NodePort: 30001,
NodePort: 30000,
NodeInterface: "eno3",
},
SSHAuthorizedKeys: []string{
@ -281,6 +275,22 @@ func CreateSIPCluster(name string, namespace string, controlPlanes int, workers
NodeSSHPrivateKeys: sshPrivateKeySecretName,
},
},
LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{
{
SIPClusterService: airshipv1.SIPClusterService{
NodeInterface: "eno3",
NodePort: 30001,
},
},
},
LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{
{
SIPClusterService: airshipv1.SIPClusterService{
NodeInterface: "eno3",
NodePort: 30002,
},
},
},
},
},
Status: airshipv1.SIPClusterStatus{},