From afe7bbf0ac88dba6695c1f1a90c725b6d8550ce5 Mon Sep 17 00:00:00 2001 From: Nate Johnston Date: Thu, 31 May 2018 12:52:52 -0400 Subject: [PATCH] adjustment of the story Change-Id: Ibf81b7d37bd724b78bd2cfd9c4e88f1c37578268 --- Makefile | 6 +- STORY.md | 29 ++-- events_json_fixtures_test.go | 16 +-- qualys/assets_test.go | 251 ----------------------------------- 4 files changed, 22 insertions(+), 280 deletions(-) delete mode 100644 qualys/assets_test.go diff --git a/Makefile b/Makefile index 60c84d4..4e2c025 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,12 @@ SOURCE?=./... env: @echo "Running build" - $(HOME)/go/bin/vgo build +# $(HOME)/go/bin/vgo build test: @echo "Running tests" - $(HOME)/go/bin/vgo test $(SOURCE) -cover +# $(HOME)/go/bin/vgo test $(SOURCE) -cover fmt: @echo "Running fmt" - go fmt $(SOURCE) +# go fmt $(SOURCE) diff --git a/STORY.md b/STORY.md index 89fc125..7efdcd7 100644 --- a/STORY.md +++ b/STORY.md @@ -102,28 +102,21 @@ compared against the intended messages. This facilitates good testing. ## Fate of the Project -The OSEL project was implemented and installed into production. There were two -problems with it. +The OSEL project was implemented and installed into production. There was a +problem with it. -The first to become visible is that there was no exponential backoff for the -AMQP connection to the OpenStack control plane's RabbitMQ. When that RabbitMQ -had issues - which was surprisingly often - OSEL would hanner away, trying to -connect to it. That would not be too much of an issue; despite what was -effectively an infinite loop, CPU usage was not extreme. The real problem was -that connection failures were logged - and logs could become several gigabytes -in a matter of hours. This was mitigated by the OpenStack operations team -rotating the logs hourly, and alerting if an hour's worth of logs exceeded a -set size. It was my intention to use one of the many [exponential backoff +There was no exponential backoff for the AMQP connection to the OpenStack +control plane's RabbitMQ. When that RabbitMQ had issues - which was +surprisingly often - OSEL would hanner away, trying to connect to it. That +would not be too much of an issue; despite what was effectively an infinite +loop, CPU usage was not extreme. The real problem was that connection failures +were logged - and logs could become several gigabytes in a matter of hours. +This was mitigated by the OpenStack operations team rotating the logs hourly, +and alerting if an hour's worth of logs exceeded a set size. It was my +intention to use one of the many [exponential backoff modules](https://github.com/cenkalti/backoff) available out there to make this more graceful. -The second - and fatal - issue is that S3 RiskFabric was not configured to -ingest from Qualys scans more than once a day. Since Qualys was already -scanning the CIDR block that corresponded to our OpenStack instances once a -day, we were essentially just adding noise to the system. The frequency of the -S3-Qualys imports could not be easily altered, and as a result the project was -shelved. - ## Remaining Work If OSEL were ever to be un-shelved, here are a few of the things that I wish I diff --git a/events_json_fixtures_test.go b/events_json_fixtures_test.go index f8574d7..1623f50 100644 --- a/events_json_fixtures_test.go +++ b/events_json_fixtures_test.go @@ -17,7 +17,7 @@ const ( "payload": { "port": { "status": "DOWN", - "binding:host_id": "oscomp-ch2-a06", + "binding:host_id": "controller", "name": "", "allowed_address_pairs": [ @@ -61,7 +61,7 @@ const ( "_context_project_id": "0b65cf220eab4a3cbd68681d188d7dc7", "_context_timestamp": "2016-10-03 18:40:34.477012", "_context_user_name": "neutron", - "publisher_id": "network.osctrl-ch2-a03", + "publisher_id": "network.controller03", "message_id": "71047538-531f-4aca-be09-a31bec441d16" } @@ -103,7 +103,7 @@ const ( "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32", "_context_timestamp":"2016-10-03 17:50:59.925462", "_context_user_name":"admin", - "publisher_id":"network.osctrl-ch2-a03", + "publisher_id":"network.controller03", "message_id":"6c93e24f-0892-494b-8e68-46252ceb9611" } ` @@ -144,7 +144,7 @@ const ( "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32", "_context_timestamp":"2016-10-03 18:05:35.769947", "_context_user_name":"admin", - "publisher_id":"network.osctrl-ch2-a03", + "publisher_id":"network.controller03", "message_id":"f67b70d5-a782-4c5e-a274-a7ff197b73ec" } ` @@ -184,7 +184,7 @@ const ( "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32", "_context_timestamp":"2016-10-03 17:32:25.665588", "_context_user_name":"admin", - "publisher_id":"network.osctrl-ch2-a03", + "publisher_id":"network.controller03", "message_id":"4df01871-8bdb-4b85-bb34-cbff59ee6034" } ` @@ -224,7 +224,7 @@ const ( "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32", "_context_timestamp":"2016-10-03 17:36:58.712962", "_context_user_name":"admin", - "publisher_id":"network.osctrl-ch2-a03", + "publisher_id":"network.controller03", "message_id":"e2d7c089-8194-4523-8f84-ae22db497f60" } ` @@ -264,7 +264,7 @@ const ( "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32", "_context_timestamp":"2016-10-03 18:09:11.876789", "_context_user_name":"admin", - "publisher_id":"network.osctrl-ch2-a03", + "publisher_id":"network.controller03", "message_id":"afb043b6-fa56-470b-b17e-984fb4cb6505" } ` @@ -292,7 +292,7 @@ const ( "_context_project_id": "ada3b9b0dbac429f9361e803b54f5f32", "_context_timestamp": "2016-10-03 18:14:32.962116", "_context_user_name": "admin", - "publisher_id": "network.osctrl-ch2-a03", + "publisher_id": "network.controller03", "message_id": "9bc5106c-a08b-4cda-9311-20bc16bc3008" } ` diff --git a/qualys/assets_test.go b/qualys/assets_test.go deleted file mode 100644 index 4e892d0..0000000 --- a/qualys/assets_test.go +++ /dev/null @@ -1,251 +0,0 @@ -package qualys - -import ( - "fmt" - "net/http" - "reflect" - "testing" -) - -func TestListAssetGroups(t *testing.T) { - - cases := []struct { - name string - response string - expected []AssetGroup - opts *ListAssetGroupOptions - isErr bool - }{ - { - name: "ListAssetGroups - single item, without list options", - response: assetGroupsXMLSingleGroup, - expected: []AssetGroup{ - { - ID: "1759735", - Title: "AG - Elastic Cloud Dynamic Perimeter", - IPs: AssetGroupIPs{ - IPs: []string{"10.1.1.1", "10.10.10.11"}, - IPRanges: nil, - }, - }, - }, - opts: nil, - }, - { - name: "ListAssetGroups - single item, with list options", - response: assetGroupsXMLSingleGroup, - expected: []AssetGroup{ - { - ID: "1759735", - Title: "AG - Elastic Cloud Dynamic Perimeter", - IPs: AssetGroupIPs{ - IPs: []string{"10.1.1.1", "10.10.10.11"}, - IPRanges: nil, - }, - }, - }, - opts: &ListAssetGroupOptions{Ids: []string{}}, - }, - { - name: "ListAssetGroups - multi item", - response: assetGroupsXMLMultiGroups, - expected: []AssetGroup{ - {ID: "1759734", Title: "AG - New"}, - {ID: "1759735", Title: "AG - Elastic Cloud Dynamic Perimeter", - IPs: AssetGroupIPs{ - IPs: []string{"10.10.10.14"}, - IPRanges: []string{"10.10.10.3-10.10.10.6"}, - }, - }, - }, - opts: &ListAssetGroupOptions{Ids: []string{"1", "2"}}, - }, - } - - for _, c := range cases { - setup() - defer teardown() - mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) { - testMethod(t, r, "GET") - fmt.Fprint(w, c.response) - }) - - assetGroups, _, err := client.Assets.ListAssetGroups(c.opts) - if err != nil { - t.Errorf("Assets.ListAssetGroups returned error: %v", err) - } - - if !reflect.DeepEqual(assetGroups, c.expected) { - t.Errorf("Assets.ListAssetGroups case: %s returned %+v, expected %+v", c.name, assetGroups, c.expected) - } - } -} - -func TestGetAssetGroupByID(t *testing.T) { - setup() - defer teardown() - - mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) { - testMethod(t, r, "GET") - fmt.Fprint(w, assetGroupsXMLSingleGroup) - }) - - groupID := "1759735" - - assetGroup, _, err := client.Assets.GetAssetGroupByID(groupID) - if err != nil { - t.Errorf("Assets.GetAssetGroupByID(%s) returned error: %v", groupID, err) - } - - expected := &AssetGroup{ - ID: "1759735", - Title: "AG - Elastic Cloud Dynamic Perimeter", - IPs: AssetGroupIPs{ - IPs: []string{"10.1.1.1", "10.10.10.11"}, - IPRanges: nil, - }, - } - if !reflect.DeepEqual(assetGroup, expected) { - t.Errorf("Assets.GetAssetGroupByID(%s) returned %+v, expected %+v", groupID, assetGroup, expected) - } -} - -func TestAddIPsToGroup(t *testing.T) { - setup() - defer teardown() - - groupID := "1759735" - ip := "10.10.10.10" - - mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) { - testMethod(t, r, "POST") - if r.FormValue("add_ips") != ip { - t.Errorf("Request form data did not include the correct IP") - } - if r.FormValue("id") != groupID { - t.Errorf("Request form data did not include the correct asset group ID") - } - fmt.Fprint(w, assetGroupsAddIPsResponse) - }) - opts := &AddIPsToGroupOptions{ - GroupID: groupID, - IPs: []string{ip}, - } - - _, err := client.Assets.AddIPsToGroup(opts) - if err != nil { - t.Errorf("Assets.AddIPsToGroup returned error: %v", err) - } -} - -func TestAssetGroupContainsIP(t *testing.T) { - cases := []struct { - name string - ip string - group *AssetGroup - expected bool - }{ - { - name: "AssetGroup.ContainsIP - nil", - ip: "10.1.1.1", - group: &AssetGroup{ID: "1759735", Title: "AG - Elastic Cloud Dynamic Perimeter"}, - expected: false, - }, - { - name: "AssetGroup.ContainsIP - empty", - ip: "10.1.1.1", - group: &AssetGroup{ - ID: "1759735", - Title: "AG - Elastic Cloud Dynamic Perimeter", - IPs: AssetGroupIPs{}}, - expected: false, - }, - { - name: "AssetGroup.ContainsIP - single item list", - ip: "10.1.1.1", - group: &AssetGroup{ - ID: "1759735", - Title: "AG - Elastic Cloud Dynamic Perimeter", - IPs: AssetGroupIPs{ - IPs: []string{"10.1.1.1"}, - IPRanges: []string{}, - }, - }, - expected: true, - }, - { - name: "AssetGroup.ContainsIP - multi item list", - ip: "10.1.1.1", - group: &AssetGroup{ - ID: "1759735", - Title: "AG - Elastic Cloud Dynamic Perimeter", - IPs: AssetGroupIPs{ - IPs: []string{"10.1.1.1"}, - IPRanges: []string{"10.10.1.1-10.10.10.10"}, - }, - }, - expected: true, - }, - } - for _, c := range cases { - contains := c.group.ContainsIP(c.ip) - if contains != c.expected { - t.Errorf("%s - AssetGroup.ContainsIP(%s) returned %v, expected %v", c.name, c.ip, contains, c.expected) - } - } -} - -func TestAssetGroupIPsContainsIP(t *testing.T) { - group := AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.10.3-10.10.10.6"}} - - cases := []struct { - name string - ip string - group AssetGroupIPs - expected bool - }{ - { - name: "AssetGroupIPs.ContainsIP - IP value match", - ip: "10.0.1.1", - group: group, - expected: true, - }, - { - name: "AssetGroupIPs.ContainsIP - IP value no match", - ip: "192.0.1.1", - group: group, - expected: false, - }, - { - name: "AssetGroupIPs.ContainsIP - IP Range value match", - ip: "10.10.10.4", - group: group, - expected: true, - }, - { - name: "AssetGroupIPs.ContainsIP - IP Range value no match", - ip: "10.10.10.1", - group: group, - expected: false, - }, - { - name: "AssetGroupIPs.ContainsIP - IP Range value match", - ip: "10.10.0.4", - group: AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.0.0-10.10.10.6"}}, - expected: true, - }, - { - name: "AssetGroupIPs.ContainsIP - IP Range value no match", - ip: "10.10.0.4", - group: AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.1.3-10.10.10.6"}}, - expected: false, - }, - } - - for _, c := range cases { - contains := c.group.ContainsIP(c.ip) - if contains != c.expected { - t.Errorf("%s - AssetGroupIPs.ContainsIP(%s) returned %v, expected %v", c.name, c.ip, contains, c.expected) - } - } -}