opendev/system-config
Code Issues Proposed changes

Merge "Retrieve logs > DEBUG level."

Browse Source
This commit is contained in:
Jenkins 2013-10-03 01:00:00 +00:00 committed by Gerrit Code Review
commit 26e9b598c9
1 changed files with 18 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
modules/openstack_project/files/logstash/log-gearman-worker.py
View File

@ -121,21 +121,29 @@ class LogRetriever(threading.Thread):
def _get_log_data(self, source_url, retry): def _get_log_data(self, source_url, retry):
gzipped = False gzipped = False
try: try:
logging.debug("Retrieving: " + source_url) # TODO(clarkb): We really should be using requests instead
r = urllib2.urlopen(source_url) # of urllib2. urllib2 will automatically perform a POST
if 'gzip' in r.info().get('Content-Type', ''): # instead of a GET if we provide urlencoded data to urlopen
gzipped = True # but we need to do a GET. The parameters are currently
# hardcoded so this should be ok for now.
logging.debug("Retrieving: " + source_url + ".gz?level=INFO")
req = urllib2.Request(source_url + ".gz?level=INFO")
req.add_header('Accept-encoding', 'gzip')
r = urllib2.urlopen(req)
except urllib2.URLError: except urllib2.URLError:
try: try:
logging.debug("Retrieving: " + source_url + ".gz") # Fallback on GETting unzipped data.
r = urllib2.urlopen(source_url + ".gz") logging.debug("Retrieving: " + source_url + "?level=INFO")
gzipped = True r = urllib2.urlopen(source_url + "?level=INFO")
except: except:
logging.exception("Unable to retrieve source file.") logging.exception("Unable to retrieve source file.")
raise raise
except: except:
logging.exception("Unable to retrieve source file.") logging.exception("Unable to retrieve source file.")
raise raise
if ('gzip' in r.info().get('Content-Type', '') or
'gzip' in r.info().get('Content-Encoding', '')):
gzipped = True
raw_buf = r.read() raw_buf = r.read()
# Hack to read all of Jenkins console logs as they upload # Hack to read all of Jenkins console logs as they upload
@ -148,9 +156,10 @@ class LogRetriever(threading.Thread):
for i in range(60): for i in range(60):
# Try for up to 60 seconds to retrieve the complete log file. # Try for up to 60 seconds to retrieve the complete log file.
try: try:
logging.debug(str(i) + " Retrying fetch of: " + source_url) logging.debug(str(i) + " Retrying fetch of: " +
source_url + "?level=INFO")
logging.debug("Fetching bytes=" + str(content_len) + '-') logging.debug("Fetching bytes=" + str(content_len) + '-')
req = urllib2.Request(source_url) req = urllib2.Request(source_url + "?level=INFO")
req.add_header('Range', 'bytes=' + str(content_len) + '-') req.add_header('Range', 'bytes=' + str(content_len) + '-')
r = urllib2.urlopen(req) r = urllib2.urlopen(req)
raw_buf += r.read() raw_buf += r.read()