From 67b28477d48cddaa1c020d544577cb162c6b41c8 Mon Sep 17 00:00:00 2001 From: Matt McEuen Date: Fri, 15 Jan 2021 12:15:46 -0600 Subject: [PATCH] Add base IPAM support ViNO needs to be able to assign IP addresses to the VMs that it instantiates, and so needs to do some light IP Address Management (IPAM). This change adds a library with an in-memory implementation that can allocate IPv4 and IPv6 addresses. Future changes will add persistance of IPAM state, the ability to de-allocate subnets/ranges/ips, and additional input validation. Change-Id: I1e2106f512f9f6fd8eb77fc032b181122158b585 --- go.mod | 2 +- pkg/ipam/errors.go | 80 ++++++++++ pkg/ipam/ipam.go | 219 ++++++++++++++++++++++++++ pkg/ipam/ipam_test.go | 353 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 653 insertions(+), 1 deletion(-) create mode 100644 pkg/ipam/errors.go create mode 100644 pkg/ipam/ipam.go create mode 100644 pkg/ipam/ipam_test.go diff --git a/go.mod b/go.mod index 4109c0a..95ae77f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/onsi/ginkgo v1.14.2 github.com/onsi/gomega v1.10.2 github.com/prometheus/common v0.10.0 - github.com/stretchr/testify v1.6.1 // indirect + github.com/stretchr/testify v1.6.1 go.uber.org/zap v1.15.0 golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/pkg/ipam/errors.go b/pkg/ipam/errors.go new file mode 100644 index 0000000..4d2c47c --- /dev/null +++ b/pkg/ipam/errors.go @@ -0,0 +1,80 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ipam + +import ( + "fmt" +) + +// ErrSubnetNotAllocated returned if the subnet is not registered in IPAM +type ErrSubnetNotAllocated struct { + Subnet string +} + +// ErrSubnetRangeOverlapsWithExistingRange returned if the subnet's range +// overlaps (partially or completely) with an already added range in that subnet +type ErrSubnetRangeOverlapsWithExistingRange struct { + Subnet string + SubnetRange Range +} + +// ErrSubnetRangeNotAllocated returned if the subnet's range is not registered in IPAM +type ErrSubnetRangeNotAllocated struct { + Subnet string + SubnetRange Range +} + +// ErrSubnetRangeExhausted returned if the subnet's range has no unallocated IPs +type ErrSubnetRangeExhausted struct { + Subnet string + SubnetRange Range +} + +// ErrInvalidIPAddress returned if an IP address string is malformed +type ErrInvalidIPAddress struct { + IP string +} + +// ErrNotSupported returned if unsupported address types are used +type ErrNotSupported struct { + Message string +} + +func (e ErrSubnetNotAllocated) Error() string { + return fmt.Sprintf("IPAM subnet %s not allocated", e.Subnet) +} + +func (e ErrSubnetRangeOverlapsWithExistingRange) Error() string { + return fmt.Sprintf("IPAM range [%s,%s] in subnet %s overlaps with an existing range", + e.SubnetRange.Start, e.SubnetRange.Stop, e.Subnet) +} + +func (e ErrSubnetRangeNotAllocated) Error() string { + return fmt.Sprintf("IPAM range [%s,%s] in subnet %s is not allocated", + e.SubnetRange.Start, e.SubnetRange.Stop, e.Subnet) +} + +func (e ErrSubnetRangeExhausted) Error() string { + return fmt.Sprintf("IPAM range [%s,%s] in subnet %s is exhausted", + e.SubnetRange.Start, e.SubnetRange.Stop, e.Subnet) +} + +func (e ErrInvalidIPAddress) Error() string { + return fmt.Sprintf("IP address %s is invalid", e.IP) +} + +func (e ErrNotSupported) Error() string { + return fmt.Sprintf("%s", e.Message) +} diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go new file mode 100644 index 0000000..13dc9b0 --- /dev/null +++ b/pkg/ipam/ipam.go @@ -0,0 +1,219 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ipam + +import ( + "net" + "strings" + "unsafe" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Ipam provides IPAM reservation, backed by IPPool CRs +type Ipam struct { + Log logr.Logger + Scheme *runtime.Scheme + Client client.Client + + ippools map[string]*IPPool +} + +// IPPool tracks allocation ranges and statuses within a specific +// subnet IPv4 or IPv6 subnet. It has a set of ranges of IPs +// within the subnet from which IPs can be allocated by IPAM, +// and a set of IPs that are currently allocated already. +type IPPool struct { + Subnet string + Ranges []Range + AllocatedIPs []string +} + +// Range has (inclusive) bounds within a subnet from which IPs can be allocated +type Range struct { + Start string + Stop string +} + +// NewIpam initializes an empty IPAM configuration. +// TODO: persist and refresh state from the API server +// TODO: add ability to remove IP addresses and ranges +func NewIpam() *Ipam { + ippools := make(map[string]*IPPool) + return &Ipam{ + ippools: ippools, + } +} + +// AddSubnetRange adds a range within a subnet for IP allocation +// TODO error: invalid range for subnet +// TODO error: range overlaps with existing range or subnet overlaps with existing subnet +func (i *Ipam) AddSubnetRange(subnet string, subnetRange Range) error { + // Does the subnet already exist? (this is fine) + ippool, exists := i.ippools[subnet] + if !exists { + ippool = &IPPool{ + Subnet: subnet, + Ranges: []Range{subnetRange}, // TODO DeepCopy() + AllocatedIPs: []string{}, + } + i.ippools[subnet] = ippool + } else { + // Does the subnet's requested range already exist? (this should fail) + exists = false + for _, r := range ippool.Ranges { + if r == subnetRange { + exists = true + break + } + } + if exists { + return ErrSubnetRangeOverlapsWithExistingRange{Subnet: subnet, SubnetRange: subnetRange} + } + } + ippool.Ranges = append(ippool.Ranges, subnetRange) + + return nil +} + +// AllocateIP allocates an IP from a range and return it +func (i *Ipam) AllocateIP(subnet string, subnetRange Range) (string, error) { + // NOTE/TODO: this is not threadsafe, which is fine because + // the final impl will use the api server as the backend, not local. + ippool, exists := i.ippools[subnet] + if !exists { + return "", ErrSubnetNotAllocated{Subnet: subnet} + } + // Make sure the range has been allocated within the subnet + var match bool + for _, r := range ippool.Ranges { + if r == subnetRange { + match = true + break + } + } + if !match { + return "", ErrSubnetRangeNotAllocated{Subnet: subnet, SubnetRange: subnetRange} + } + + ip, err := findFreeIPInRange(ippool, subnetRange) + if err != nil { + return "", err + } + ippool.AllocatedIPs = append(ippool.AllocatedIPs, ip) + return ip, nil +} + +// This converts IP ranges/addresses into iterable ints, +// steps through them looking for one that that is not already +// in use, converts it back to a string and returns it. +// It does not itself add it to the list of assigned IPs. +func findFreeIPInRange(ippool *IPPool, subnetRange Range) (string, error) { + allocatedIPSet := sliceToMap(ippool.AllocatedIPs) + intToString := intToIPv4String + if strings.Contains(ippool.Subnet, ":") { + intToString = intToIPv6String + } + + // Step through the range looking for free IPs + start, err := ipStringToInt(subnetRange.Start) + if err != nil { + return "", err + } + stop, err := ipStringToInt(subnetRange.Stop) + if err != nil { + return "", err + } + + for ip := start; ip <= stop; ip++ { + _, in := allocatedIPSet[intToString(ip)] + if !in { + // Found an unallocated IP + return intToString(ip), nil + } + } + return "", ErrSubnetRangeExhausted{ippool.Subnet, subnetRange} +} + +// Create a map[string]struct{} representation of a string slice, +// for efficient set lookups +func sliceToMap(slice []string) map[string]struct{} { + m := map[string]struct{}{} + for _, s := range slice { + m[s] = struct{}{} + } + return m +} + +// Convert an IPV4 or IPV6 address string to an easily iterable uint64. +// For IPV4 addresses, this captures the full address (padding the MSB with 0's) +// For IPV6 addresses, this captures the most significant 8 bytes, +// and excludes the 8-byte interface identifier. +func ipStringToInt(ipString string) (uint64, error) { + ip := net.ParseIP(ipString) + if ip == nil { + return 0, ErrInvalidIPAddress{ipString} + } + + var bytes []byte + if ip.To4() != nil { + // IPv4 + bytes = append(make([]byte, 4), ip.To4()...) + } else { + // IPv6 + bytes = ip.To16()[:8] + } + + return byteArrayToInt(bytes), nil +} + +func intToIPv4String(i uint64) string { + bytes := intToByteArray(i) + ip := net.IPv4(bytes[4], bytes[5], bytes[6], bytes[7]) + return ip.String() +} + +func intToIPv6String(i uint64) string { + // Pad with 8 more bytes of zeros on the right for the hosts's interface, + // which will not be determined by IPAM. + bytes := append(intToByteArray(i), make([]byte, 8)...) + var ip net.IP = bytes + return ip.String() +} + +// Convert an uint64 into 8 bytes, with most significant byte first +// Based on https://gist.github.com/ecoshub/5be18dc63ac64f3792693bb94f00662f +func intToByteArray(num uint64) []byte { + size := 8 + arr := make([]byte, size) + for i := 0; i < size; i++ { + byt := *(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&num)) + uintptr(i))) + arr[size-i-1] = byt + } + return arr +} + +// Convert an 8-byte array to an uint64 +// Based on https://gist.github.com/ecoshub/5be18dc63ac64f3792693bb94f00662f +func byteArrayToInt(arr []byte) uint64 { + val := uint64(0) + size := 8 + for i := 0; i < size; i++ { + *(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&val)) + uintptr(size-i-1))) = arr[i] + } + return val +} diff --git a/pkg/ipam/ipam_test.go b/pkg/ipam/ipam_test.go new file mode 100644 index 0000000..f9b2450 --- /dev/null +++ b/pkg/ipam/ipam_test.go @@ -0,0 +1,353 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ipam + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAllocateIP(t *testing.T) { + tests := []struct { + name, subnet, expectedErr string + subnetRange Range + }{ + { + name: "success ipv4", + subnet: "10.0.0.0/16", + subnetRange: Range{"10.0.1.0", "10.0.1.9"}, + }, + { + name: "success ipv6", + subnet: "2600:1700:b030:0000::/72", + subnetRange: Range{"2600:1700:b030:0000::", "2600:1700:b030:0009::"}, + }, + { + name: "error subnet not allocated ipv4", + subnet: "10.0.0.0/20", + subnetRange: Range{"10.0.1.0", "10.0.1.9"}, + expectedErr: "IPAM subnet 10.0.0.0/20 not allocated", + }, + { + name: "error subnet not allocated ipv6", + subnet: "2600:1700:b030:0000::/80", + subnetRange: Range{"2600:1700:b030:0000::", "2600:1700:b030:0009::"}, + expectedErr: "IPAM subnet 2600:1700:b030:0000::/80 not allocated", + }, + { + name: "error range not allocated ipv4", + subnet: "10.0.0.0/16", + subnetRange: Range{"10.0.2.0", "10.0.2.9"}, + expectedErr: "IPAM range [10.0.2.0,10.0.2.9] in subnet 10.0.0.0/16 is not allocated", + }, + { + name: "error range not allocated ipv6", + subnet: "2600:1700:b030:0000::/72", + subnetRange: Range{"2600:1700:b030:0000::", "2600:1700:b030:1111::"}, + expectedErr: "IPAM range [2600:1700:b030:0000::,2600:1700:b030:1111::] " + + "in subnet 2600:1700:b030:0000::/72 is not allocated", + }, + { + name: "error range exhausted ipv4", + subnet: "192.168.0.0/1", + subnetRange: Range{"192.168.0.0", "192.168.0.0"}, + expectedErr: "IPAM range [192.168.0.0,192.168.0.0] in subnet 192.168.0.0/1 is exhausted", + }, + { + name: "error range exhausted ipv6", + subnet: "2600:1700:b031:0000::/64", + subnetRange: Range{"2600:1700:b031:0000::", "2600:1700:b031:0000::"}, + expectedErr: "IPAM range [2600:1700:b031:0000::,2600:1700:b031:0000::] " + + "in subnet 2600:1700:b031:0000::/64 is exhausted", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ipammer := NewIpam() + + // Pre-populate IPAM with some precondition test data + err := ipammer.AddSubnetRange("10.0.0.0/16", Range{"10.0.1.0", "10.0.1.9"}) + require.NoError(t, err) + err = ipammer.AddSubnetRange("2600:1700:b030:0000::/72", Range{"2600:1700:b030:0000::", "2600:1700:b030:0009::"}) + require.NoError(t, err) + err = ipammer.AddSubnetRange("192.168.0.0/1", Range{"192.168.0.0", "192.168.0.0"}) + require.NoError(t, err) + err = ipammer.AddSubnetRange("2600:1700:b031:0000::/64", Range{"2600:1700:b031:0000::", "2600:1700:b031:0000::"}) + require.NoError(t, err) + _, err = ipammer.AllocateIP("192.168.0.0/1", Range{"192.168.0.0", "192.168.0.0"}) + require.NoError(t, err) + _, err = ipammer.AllocateIP("2600:1700:b031:0000::/64", Range{"2600:1700:b031:0000::", "2600:1700:b031:0000::"}) + require.NoError(t, err) + ip, err := ipammer.AllocateIP(tt.subnet, tt.subnetRange) + if tt.expectedErr != "" { + assert.Equal(t, "", ip) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedErr) + } else { + require.NoError(t, err) + assert.NotEmpty(t, ip) + } + }) + } +} + +// Test some error handling that is not captured by TestAllocateIP +func TestAddSubnetRange(t *testing.T) { + tests := []struct { + name, subnet, expectedErr string + subnetRange Range + }{ + { + name: "success", + subnet: "10.0.0.0/16", + subnetRange: Range{"10.0.2.0", "10.0.2.9"}, + expectedErr: "", + }, + { + name: "error range already exists", + subnet: "10.0.0.0/16", + subnetRange: Range{"10.0.1.0", "10.0.1.9"}, + expectedErr: "IPAM range [10.0.1.0,10.0.1.9] in subnet 10.0.0.0/16 overlaps", + }, + // TODO: check for partially overlapping ranges and subnets + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ipammer := NewIpam() + + // Pre-populate IPAM with some precondition test data + err := ipammer.AddSubnetRange("10.0.0.0/16", Range{"10.0.1.0", "10.0.1.9"}) + require.NoError(t, err) + err = ipammer.AddSubnetRange(tt.subnet, tt.subnetRange) + if tt.expectedErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedErr) + } else { + require.NoError(t, err) + } + }) + } +} +func TestFindFreeIPInRange(t *testing.T) { + tests := []struct { + name string + subnet string + subnetRange Range + out string + expectedErr string + }{ + { + name: "ip available IPv4", + subnet: "10.0.0.0/16", + subnetRange: Range{"10.0.1.0", "10.0.1.10"}, + out: "10.0.1.0", + }, + { + name: "ip unavailable IPv4", + subnet: "10.0.0.0/16", + subnetRange: Range{"10.0.2.0", "10.0.2.0"}, + out: "", + expectedErr: "IPAM range [10.0.2.0,10.0.2.0] in subnet 10.0.0.0/16 is exhausted", + }, + { + name: "ip available IPv6", + subnet: "2600:1700:b030:0000::/64", + subnetRange: Range{"2600:1700:b030:1001::", "2600:1700:b030:1009::"}, + out: "2600:1700:b030:1001::", + }, + { + name: "ip unavailable IPv6", + subnet: "2600:1700:b031::/64", + subnetRange: Range{"2600:1700:b031::", "2600:1700:b031::"}, + expectedErr: "IPAM range [2600:1700:b031::,2600:1700:b031::] " + + "in subnet 2600:1700:b031::/64 is exhausted", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ippool := IPPool{ + Subnet: tt.subnet, + // One available and one unavailable range each for ipv4/6 + Ranges: []Range{ + {"10.0.1.0", "10.0.1.10"}, + {"10.0.2.0", "10.0.2.0"}, + {"2600:1700:b030:1001::", "2600:1700:b030:1009::"}, + {"2600:1700:b031::", "2600:1700:b031::"}, + }, + AllocatedIPs: []string{"10.0.2.0", "2600:1700:b031::"}, + } + actual, err := findFreeIPInRange(&ippool, tt.subnetRange) + if tt.expectedErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedErr) + } else { + require.NoError(t, err) + assert.Equal(t, tt.out, actual) + } + }) + } +} + +func TestSliceToMap(t *testing.T) { + tests := []struct { + name string + in []string + out map[string]struct{} + }{ + { + name: "empty slice", + in: []string{}, + out: map[string]struct{}{}, + }, + { + name: "one-element slice", + in: []string{"foo"}, + out: map[string]struct{}{"foo": {}}, + }, + { + name: "two-element slice", + in: []string{"foo", "bar"}, + out: map[string]struct{}{"foo": {}, "bar": {}}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + actual := sliceToMap(tt.in) + assert.Equal(t, tt.out, actual) + }) + } +} + +func TestIPStringToInt(t *testing.T) { + tests := []struct { + name string + in string + out uint64 + expectedErr string + }{ + { + name: "valid IPv4 address", + in: "1.0.0.1", + out: uint64(math.Pow(2, 24) + 1), + }, + { + name: "invalid IPv4 address", + in: "1.0.0.1.1", + out: 0, + expectedErr: " is invalid", + }, + { + name: "valid IPv6 address", + in: "0001:0000:0000:0001::", + out: uint64(math.Pow(2, 48) + 1), + }, + { + name: "invalid IPv6 address", + in: "1000:0000:0000:foobar::", + out: 0, + expectedErr: " is invalid", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + actual, err := ipStringToInt(tt.in) + if tt.expectedErr != "" { + require.Error(t, err) + assert.Empty(t, tt.out) + assert.Contains(t, err.Error(), tt.expectedErr) + } else { + require.NoError(t, err) + assert.Equal(t, tt.out, actual) + } + }) + } +} + +func TestIntToByteArray(t *testing.T) { + tests := []struct { + name string + in uint64 + out []byte + }{ + { + name: "zeros", + in: 0, + out: make([]byte, 8), + }, + { + name: "IPv4 255's", + in: uint64(math.Pow(2, 32) - 1), + out: []byte{0, 0, 0, 0, 255, 255, 255, 255}, + }, + { + name: "value in the middle", + in: 512, + out: []byte{0, 0, 0, 0, 0, 0, 2, 0}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + actual := intToByteArray(tt.in) + assert.Equal(t, tt.out, actual) + }) + } +} + +func TestByteArrayToInt(t *testing.T) { + tests := []struct { + name string + in []byte + out uint64 + }{ + { + name: "zeros", + in: make([]byte, 8), + out: 0, + }, + { + name: "255's", + in: []byte{0, 0, 0, 0, 255, 255, 255, 255}, + out: uint64(math.Pow(2, 32) - 1), + }, + { + name: "value in the middle", + in: []byte{0, 0, 0, 0, 0, 0, 2, 0}, + out: 512, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + actual := byteArrayToInt(tt.in) + assert.Equal(t, tt.out, actual) + }) + } +}