Add Verification for List Queue

This patch adds more verification for List Queue API call.
It also cleans up some files, to keep the style consistent.

blueprint system-tests

Change-Id: Ieda206a98f22e7642784122b5c7b8fd45f2fd39e

README.rst

@@ -5,14 +5,16 @@
 1. `Install mongodb`_ locally
 2. Start your local MongoDB instance::
 
-    	mongod
+       mongod
+
 3. Clone the Marconi repo::
 
-  	git clone https://github.com/stackforge/marconi.git
+       git clone https://github.com/stackforge/marconi.git
+
 4. cd to your local copy of the repo
 5. Copy the Marconi config files to the directory ~/.marconi::
 
-  	 cp -r marconi/etc/*.conf-sample ~/.marconi
+       cp -r marconi/etc/*.conf-sample ~/.marconi
 6. Update the [drivers:storage:mongodb] section in ~/.marconi/marconi.conf
 
    6a. Comment out the uri pointing to replicaset::

marconi/tests/system/claim/claim_tests.txt

@@ -1,5 +1,5 @@
-| *Setting*      | *Value* 			    		|
+| *Setting*      | *Value*                      |
-| Documentation  | Marconi - Queue Test Suite	|
+| Documentation  | Marconi - Queue Test Suite   |
 | Library        | ../common/http.py            |
 | Library        | ../common/functionlib.py     |
 | Library        | ../messages/msgfnlib.py      |
@@ -13,7 +13,7 @@
 
 | *Test Case*        | *Action*            | *Argument*          | *Argument*         | *Argument*    | # Comment
 | 0:SUITE SETUP      | [DOCUMENTATION]     |                     | Post 500 messages  |               | #SUITE SETUP - POST MULTIPLE MESSAGES
-|                    | [Tags]			   | INSERT_MESSAGE      |                    |               | #(Robot allows only one keyword in setup)
+|                    | [Tags]              | INSERT_MESSAGE      |                    |               | #(Robot allows only one keyword in setup)
 |                    | ${reqparam}=        | Create Dictionary   | messagecount       | ${50}         | # Specify count of messages to be posted
 |                    |                     | ...                 | ttl                | ${300}        | # Specify count of messages to be posted
 |                    | ${msgbody}=         | dummygetmessagebody | ${reqparam}        |               | # Gets the message body to post
@@ -29,7 +29,7 @@
 |                    | @{postresponse}=    | executetests    | ${API_TEST_DATA[1]}    | # Post a claim; postresponse = [httpheaders, httpresponsebody]
 |                    | verifyclaimmsg      | ${5}            | @{postresponse}        |
 | 3:CLAIM   MESSAGES | [DOCUMENTATION]     | Claim messages  |                        | #TEST CASE 3 - CLAIM DEFAULT # OF MESSAGES
-|                    | ...                 | with no params  |                        |                (currently 10) 
+|                    | ...                 | with no params  |                        |                (currently 10)
 |                    | @{postresponse}=    | executetests    | ${API_TEST_DATA[2]}    | # postresponse = [httpheaders, httpresponsebody]
 |                    | verifyclaimmsg      | ${10}           | @{postresponse}        |
 | 4:CLAIM 15 MESSAGE | [DOCUMENTATION]     | Claim messages  |                        | #TEST CASE 4 - CLAIM 15 MESSAGES
@@ -51,7 +51,7 @@
 |                    | @{postresponse}=    | executetests    | ${API_TEST_DATA[9]}    | # Post a claim with TTL= 1 sec
 |                    | Sleep               | 3s              |                        |
 |                    | patchclaim          | @{postresponse} |                        | # Patch the above claim
-| 9: DELETE MESSAGE ON EXPIRED CLAIM  | [DOCUMENTATION]      | Delete message         | # TEST CASE 9 - DELETE MESSAGE ON AN EXPIRED CLAIM 
+| 9: DELETE MESSAGE ON EXPIRED CLAIM  | [DOCUMENTATION]      | Delete message         | # TEST CASE 9 - DELETE MESSAGE ON AN EXPIRED CLAIM
 |                    | ...                                   | on expired claim       |
 |                    | @{postresponse}=    | executetests    | ${API_TEST_DATA[10]}   | # Post a claim with TTL= 1 sec
 |                    | Sleep               | 3s              |                        |
@@ -59,7 +59,7 @@
 | 10: RELEASE CLAIM  | [DOCUMENTATION]     | Release claim   |                        | # TEST CASE 10 - RELEASE CLAIM
 |                    | @{postresponse}=    | executetests    | ${API_TEST_DATA[11]}   | # Post a claim with TTL= 1 sec
 |                    | releaseclaim        | @{postresponse} |                        | # Patch the above claim
-| 11: GET MESSAGE FROM EXPIRED CLAIM  | [DOCUMENTATION]      | Get message            | # TEST CASE 11 - GET MESSAGE FROM EXPIRED CLAIM 
+| 11: GET MESSAGE FROM EXPIRED CLAIM  | [DOCUMENTATION]      | Get message            | # TEST CASE 11 - GET MESSAGE FROM EXPIRED CLAIM
 |                    | ...                                   | from expired claim     |
 |                    | @{postresponse}=    | executetests    | ${API_TEST_DATA[12]}   | # Post a claim with TTL= 1 sec
 |                    | Sleep               | 3s              |                        |

marconi/tests/system/claim/claimfnlib.py

@@ -23,10 +23,10 @@ def verify_claim_msg(count, *claim_response):
     """Verifies claim messages.
 
     Validation steps include - verifying the
-    1. number of messages returned is <= limit specified
+    1. number of messages returned is <= limit specified.
-    2. query claim & verifying the response
+    2. query claim & verifying the response.
-    :param count: limit specified in the claim request
+    :param count: limit specified in the claim request.
-    :param claim_response : response returned for the claim request
+    :param claim_response : [header, body] returned for post claim request.
     """
     msg_length_flag = False
 
@@ -41,7 +41,11 @@ def verify_claim_msg(count, *claim_response):
 
 
 def verify_claim_msglength(count, *body):
-    """Validates that number of messages returned is <= limit specified."""
+    """Validates that number of messages returned is <= limit specified.
+
+    :param count: value of limit specified in the post claim.
+    :param *body: response body returned for the post claim.
+    """
     msg_list = body
     msg_list = json.loads(msg_list[0])
     return (len(msg_list) <= count)
@@ -53,6 +57,8 @@ def query_claim(headers, *body):
     Does a Query Claim using the href in post claim.
     Compares the messages returned in Query claim with the messages
     returned on Post Claim.
+    :param headers: headers returned in the post claim response.
+    :param *body: message list returned in the post claim response.
     """
     test_result_flag = False
 
@@ -90,6 +96,8 @@ def verify_query_msgs(querymsgs, msg_list):
 
     Compares the messages returned in Query Claim with the messages
     returned when the claim was posted.
+    :param querymsgs: response body returned for Query Claim.
+    :param msg_list: message list returned for the original claim.
     """
     test_result_flag = True
     idx = 0
@@ -109,6 +117,7 @@ def patch_claim(*claim_response):
 
     Extracts claim id from the POST response input & updates the claim.
     If PATCH claim succeeds, verifies that the claim TTL is extended.
+    :param *claim_response: [headers, body] returned for the original claim
     """
     test_result_flag = False
 
@@ -161,7 +170,7 @@ def verify_patch_claim(url, header, ttl_extended):
 def create_urllist_fromhref(*response):
     """EXtracts href & creates a url list.
 
-    :param *response : http response text with the list of messages.
+    :param *response : http response containing the list of messages.
     """
     rspbody = json.loads(response[1])
     urllist = [functionlib.create_url_from_appender(item["href"])
@@ -173,6 +182,7 @@ def delete_claimed_msgs(*claim_response):
     """Deletes claimed messages.
 
     Verifies that the deletes were successful with a GET on the deleted msg.
+    :param *claim_response: [header, body] returned for post claim request.
     """
     test_result_flag = False
 
@@ -195,7 +205,10 @@ def delete_claimed_msgs(*claim_response):
 
 
 def get_claimed_msgs(*claim_response):
-    """Does get on all messages returned in the claim."""
+    """Does get on all messages returned in the claim.
+
+    :param *claim_response: [header, body] returned for post claim request.
+    """
     test_result_flag = True
 
     urllist = create_urllist_fromhref(*claim_response)
@@ -218,6 +231,7 @@ def release_claim(*claim_response):
 
     Extracts claim id from the POST response input & deletes the claim.
     If DELETE claim succeeds, verifies that a GET claim returns 404.
+    :param *claim_response: [header, body] returned for post claim request.
     """
     test_result_flag = False
 

marconi/tests/system/claim/getdata.py

@@ -24,14 +24,16 @@ cfg = config.Config()
 
 def get_data():
     """Reads the test data from claim/test_data.csv."""
-    DATA = []
+    data = []
     with open('marconi/tests/system/claim/test_data.csv', 'rb') as datafile:
         testdata = csv.DictReader(datafile, delimiter='|')
         for row in testdata:
-            DATA.append(row)
+            data.append(row)
-    for row in DATA:
+
+    for row in data:
         row['header'] = functionlib.get_headers(row['header'])
         row['url'] = row['url'].replace("<BASE_URL>", cfg.base_url)
-    return DATA
+
+    return data
 
 API_TEST_DATA = get_data()

marconi/tests/system/common/functionlib.py

@@ -43,6 +43,7 @@ def get_keystone_token():
     response_body = json.loads(response.text)
 
     auth_token = response_body["access"]["token"]["id"]
+
     return auth_token
 
 
@@ -52,6 +53,7 @@ def get_auth_token():
         auth_token = get_keystone_token()
     else:
         auth_token = "notrealtoken"
+
     return auth_token
 
 
@@ -138,6 +140,7 @@ def get_headers(input_header):
         header = input_header
     else:
         header = create_marconi_headers()
+
     return header
 
 
@@ -147,6 +150,7 @@ def get_custom_body(kwargs):
     if "metadatasize" in kwargs.keys():
         random_data = binascii.b2a_hex(os.urandom(kwargs["metadatasize"]))
         req_body["data"] = random_data
+
     return json.dumps(req_body)
 
 
@@ -164,13 +168,14 @@ def get_url_from_location(header):
 
 
 def verify_metadata(get_data, posted_body):
-    """@todo(malini) - Really verify the metadata."""
+    """TODO(malini) - Really verify the metadata."""
     test_result_flag = False
 
     get_data = str(get_data)
     posted_body = str(posted_body)
     print(get_data, type(get_data))
     print(posted_body, type(posted_body))
+
     if get_data in posted_body:
         print("AYYY")
     else:

marconi/tests/system/common/http.py

@@ -168,9 +168,11 @@ def verify_response(response, expected_RC):
     test_result_flag = True
     actual_RC = response.status_code
     actual_response_body = response.text
+
     if actual_RC != expected_RC:
         test_result_flag = False
         print("Unexpected http Response code {}".format(actual_RC))
         print "Response Body returned"
         print actual_response_body
+
     return test_result_flag

marconi/tests/system/messages/getdata.py

@@ -23,16 +23,18 @@ cfg = config.Config()
 
 
 def get_data():
-    """Gets Test Data from a csv file."""
+    """Gets Test data from a csv file."""
-    DATA = []
+    data = []
     with open('marconi/tests/system/messages/test_data.csv', 'rb') as datafile:
         test_data = csv.DictReader(datafile, delimiter='|')
         for row in test_data:
-            DATA.append(row)
+            data.append(row)
-    for row in DATA:
+
+    for row in data:
         row['header'] = functionlib.get_headers(row['header'])
         row['url'] = row['url'].replace("<BASE_URL>", cfg.base_url)
-    return DATA
+
+    return data
 
 
 API_TEST_DATA = get_data()

marconi/tests/system/messages/messages_tests.txt

@@ -1,7 +1,7 @@
-| *Setting*      | *Value* 			    		|
+| *Setting*      | *Value*                      |
 | Documentation  | Marconi - Message Test Suite	|
 | Library        | ../common/http.py            |
-| Library        | ../common/functionlib.py |
+| Library        | ../common/functionlib.py     |
 | Library        | msgfnlib.py                  |
 | Library        | Collections                  |
 | Variables      | getdata.py                   |
@@ -11,7 +11,7 @@
 
 | *Test Case*           | *Action*            | *Argument*      | *Argument*                  | *Argument*    | # Comment
 | 1:POST SINGLE MESSAGE | [DOCUMENTATION]     |                 | Post single message         |               | #TEST CASE 1 - POST SINGLE MESSAGE
-|                       | [Tags]			  | INSERT_MESSAGE  |                             |               |
+|                       | [Tags]              | INSERT_MESSAGE  |                             |               |
 |                       | ${reqparam}=        | Create Dictionary   | messagecount            | ${1}          | # test Setup- Specify count of messages to be posted
 |                       | ${msgbody}=         | dummygetmessagebody | ${reqparam}             |               | # test Setup- Get the message body to post
 |                       | Set To Dictionary   | ${API_TEST_DATA[1]} | body                    | ${msgbody}    | # test Setup -Set the POST body
@@ -19,7 +19,7 @@
 |                       | verifypostmsg       | ${postresponse[0]} | ${API_TEST_DATA[1]["body"]} |            | # GET the posted message & verify metadata
 |                       |
 | 2:POST MULTIPLE MESSAGES | [DOCUMENTATION]  |                 | Post 50 messages            |               | #TEST CASE 2 - POST MULTIPLE MESSAGES
-|                       | [Tags]			  | INSERT_MESSAGE  |                             |               |
+|                       | [Tags]              | INSERT_MESSAGE  |                             |               |
 |                       | ${reqparam}=        | Create Dictionary | messagecount              | ${50}         | # Specify count of messages to be posted
 |                       | ${msgbody}=         | dummygetmessagebody | ${reqparam}             |               | # Gets the message body to post
 |                       | Set To Dictionary   | ${API_TEST_DATA[2]} | body                    | ${msgbody}    | # Set the POST body
@@ -45,7 +45,7 @@
 |                       | deletemsg           | ${postresponse[0]} |                          |               | # GET the posted message & verify metadata
 | 7:POST 60 MESSAGES    | [DOCUMENTATION]     | POST > MAX NUMBER OF MESSAGES |               |               | #TEST CASE 7 - POST > 50 MESSAGES
 |                       | ...                 | ALLOWED PER POST (currently 50) |             |               |
-|                       | [Tags]			  | INSERT_MESSAGE      |                         |               |
+|                       | [Tags]              | INSERT_MESSAGE      |                         |               |
 |                       | ${reqparam}=        | Create Dictionary   | messagecount            | ${60}         | # test Setup- Specify count of messages to be posted
 |                       | ${msgbody}=         | dummygetmessagebody | ${reqparam}             |               | # test Setup- Get the message body to post
 |                       | Set To Dictionary   | ${API_TEST_DATA[6]} | body                    | ${msgbody}    | # test Setup -Set the POST body

marconi/tests/system/messages/msgfnlib.py

@@ -26,7 +26,10 @@ cfg = config.Config()
 
 
 def generate_dict(dict_length):
-    """Returns dictionary of specified length. Key:Value is random data."""
+    """Returns dictionary of specified length. Key:Value is random data.
+
+    :param dict_length: length of the dictionary
+    """
     with open('/usr/share/dict/words', 'rt') as f:
         words = f.readlines()
     words = [w.rstrip() for w in words]
@@ -43,8 +46,8 @@ def single_message_body(**kwargs):
 
     The ttl will be a random value (60 <= TTL <= 1209600).
     The message body will be random dict.
-    :param **kwargs can be {messagesize: x} , where x is message size
+    :param **kwargs: can be {messagesize: x} , where x is message size
-    :param **kwargs can be {ttl: x} , where x is ttl in seconds
+    :param **kwargs: can be {ttl: x} , where x is ttl in seconds
     """
     valid_ttl = random.randint(60, 1209600)
 
@@ -65,15 +68,13 @@ def single_message_body(**kwargs):
 def get_message_body(**kwargs):
     """Returns request body for post message tests.
 
-    :param **kwargs can be {messagecount: x} , where x is the # of messages.
+    :param **kwargs: can be {messagecount: x} , where x is the # of messages.
     """
     message_count = kwargs["messagecount"]
     multiple_message_body = []
-    i = 0
+    for i in range[message_count]:
-    while i < message_count:
         message_body = single_message_body(**kwargs)
         multiple_message_body.append(message_body)
-        i = i + 1
     return multiple_message_body
 
 
@@ -107,7 +108,10 @@ def verify_msg_length(count=10, *msg_list):
 
 
 def get_href(*msg_list):
-    """Extracts href."""
+    """Extracts href.
+
+    :param *msg_list: list of messages returned by the server.
+    """
     msg_body = json.loads(msg_list[0])
     link = msg_body["links"]
     href = link[0]["href"]
@@ -118,6 +122,8 @@ def verify_post_msg(msg_headers, posted_body):
     """Verifies the response of POST Message(s).
 
     Retrieves the posted Message(s) & validates the message metadata.
+    :param msg_headers: headers returned for post message request.
+    :param posted_body: message metadata(s) in the post message request.
     """
     test_result_flag = False
 
@@ -162,7 +168,11 @@ def get_next_msgset(responsetext):
 
 
 def verify_get_msgs(count, *getresponse):
-    """Verifies GET message & does a recursive GET if needed."""
+    """Verifies GET message & does a recursive GET if needed.
+
+    :param count: limit value specified in the get message request.
+    :param *getresponse: [headers, body] returned for get message request.
+    """
     test_result_flag = False
 
     body = getresponse[1]
@@ -179,7 +189,10 @@ def verify_get_msgs(count, *getresponse):
 
 
 def delete_msg(*postresponse):
-    """Post DELETE message & verifies that a subsequent GET returns 404."""
+    """Post DELETE message & verifies that a subsequent GET returns 404.
+
+    :param *postresponse: [headers, body] returned for post message request.
+    """
     test_result_flag = False
     headers = str(postresponse[0])
     headers = headers.replace("'", '"')

marconi/tests/system/queue/getdata.py

@@ -23,16 +23,17 @@ cfg = config.Config()
 
 
 def get_data():
-    """Gets Test Data from a csv file."""
+    """Gets Test data from a csv file."""
-    DATA = []
+    data = []
     with open('marconi/tests/system/queue/test_data.csv', 'rb') as datafile:
         test_data = csv.DictReader(datafile, delimiter='|')
         for row in test_data:
-            DATA.append(row)
+            data.append(row)
-    for row in DATA:
+
+    for row in data:
         row['header'] = functionlib.get_headers(row['header'])
         row['url'] = row['url'].replace("<BASE_URL>", cfg.base_url)
-    return DATA
 
+    return data
 
 API_TEST_DATA = get_data()

marconi/tests/system/queue/queue_tests.txt

@@ -1,4 +1,4 @@
-| *Setting*      | *Value* 			    		|
+| *Setting*      | *Value*                      |
 | Documentation  | Marconi - Queue Test Suite   |
 | Library        | ../common/http.py            |
 | Library        | queuefnlib.py                |

marconi/tests/system/queue/queuefnlib.py

@@ -18,6 +18,7 @@ import json
 import os
 
 from marconi.tests.system.common import functionlib
+from marconi.tests.system.common import http
 
 
 def verify_queue_stats(*get_response):
@@ -27,6 +28,7 @@ def verify_queue_stats(*get_response):
        1. stats json body has the keys - action & messages.
        2. messages json has the keys - claimed & free.
        3. claimed & free key values are int.
+    :param *getresponse: [headers, body] returned for get queue stats.
     """
 
     test_result_flag = True
@@ -63,6 +65,7 @@ def get_queue_name(namelength=65):
     """Returns a queuename of specified length.
 
     By default, a name longer than Marconi allows - currently 64 char.
+    :param namelength: length of the queue name.
     """
 
     appender = "/queues/" + binascii.b2a_hex(os.urandom(namelength))
@@ -71,9 +74,11 @@ def get_queue_name(namelength=65):
 
 
 def verify_list_queues(*list_queue_response):
-    """Verifies the response of list queues."""
+    """Verifies the response of list queues.
-    response_body = json.loads(list_queue_response[1])
 
+    :param *list_queue_response: [header, body] returned for list queue.
+    """
+    response_body = json.loads(list_queue_response[1])
     links = response_body['links']
     href = links[0]['href']
     detail_enabled = 'detailed=true' in href
@@ -87,9 +92,16 @@ def verify_list_queues(*list_queue_response):
         print 'List Queue API response: {}'.format(response_body)
         assert test_result_flag, 'List Queue failed'
 
+    if links[0]['rel'] == 'next':
+        list_queues(href)
+
 
 def verify_listed(queue, detail_enabled):
-    '''Verifies the listed queues.'''
+    """Verifies the listed queues.
+
+    :param queue: queue returned in the list queues response.
+    :param detail_enabled: indicates if queue contains metadata
+    """
     test_result_flag = True
 
     keys = queue.keys()
@@ -108,3 +120,24 @@ def verify_listed(queue, detail_enabled):
         test_result_flag = False
 
     return test_result_flag
+
+
+def list_queues(href):
+    """Lists queue using the href value.
+
+    :param href: href returned by a previous list queue request.
+    """
+    test_result_flag = False
+
+    url = functionlib.create_url_from_appender(href)
+    header = functionlib.create_marconi_headers()
+
+    list_queue_response = http.get(url, header)
+    if list_queue_response.status_code == 200:
+        headers = list_queue_response.headers
+        text = list_queue_response.text
+        verify_list_queues(headers, text)
+    elif list_queue_response.status_code == 204:
+        test_result_flag = True
+    else:
+        assert test_result_flag, 'List Queue failed'

marconi/tests/system/queue/test_data.csv

@@ -18,7 +18,7 @@ TestID|httpverb|url|header|body|params|expectedRC|expectedResponseBody
 14|PUT    |<BASE_URL>/queues/qtestqueue||||204|
 15|PUT    |||{"messages":{"ttl": 86400}}||400|
 16|GET    |<BASE_URL>/queues/qtestqueue/stats||||200|
-17|GET    |<BASE_URL>/queues||||200|
+17|GET    |<BASE_URL>/queues?limit=1||||200|
 18|GET    |<BASE_URL>/queues?detailed=true||||200|
 19|DELETE |<BASE_URL>/queues/qtestqueue||||204|
 20|PUT    |<BASE_URL>/queues/qtestqueue||{"messages":{"ttl": 86400}}||201|