From afe1a99f742370d5840dfe787b901f908b68ecd1 Mon Sep 17 00:00:00 2001 From: Logan V Date: Mon, 10 Oct 2016 09:40:47 -0500 Subject: [PATCH] Import osa-gate-profile A few cobbled together scripts to perform analytics on OSA gate check timings based on console logs retrieved from logstash.openstack.org. Change-Id: I5f7664482751386cca2697954573a55be06e6b9b --- osa-gate-profile/branchprofile.php | 103 ++++++++++++++++++++++++++ osa-gate-profile/fetchlogs.php | 107 +++++++++++++++++++++++++++ osa-gate-profile/generatereports.php | 39 ++++++++++ osa-gate-profile/parselogs.php | 66 +++++++++++++++++ osa-gate-profile/readme.rst | 27 +++++++ 5 files changed, 342 insertions(+) create mode 100644 osa-gate-profile/branchprofile.php create mode 100644 osa-gate-profile/fetchlogs.php create mode 100644 osa-gate-profile/generatereports.php create mode 100644 osa-gate-profile/parselogs.php create mode 100644 osa-gate-profile/readme.rst diff --git a/osa-gate-profile/branchprofile.php b/osa-gate-profile/branchprofile.php new file mode 100644 index 00000000..2c9e4bbc --- /dev/null +++ b/osa-gate-profile/branchprofile.php @@ -0,0 +1,103 @@ +#!/usr/bin/php + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +$fetch_results = 10000; +$search_days = 14; + +$search = '{"query":{"filtered":{"filter":{"bool":{"must":[{"range":{"@timestamp":{"from":1463241312439,"to":1463846112439}}},{"fquery":{"query":{"query_string":{"query":"project:(\"openstack\/openstack-ansible\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"build_name:(\"gate-openstack-ansible-dsvm-commit\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"message:(\"- Operation: [\u00a0openstack-ansible --forks\")"}},"_cache":true}}]}}}},"size":10000,"sort":[{"@timestamp":{"order":"desc","ignore_unmapped":true}},{"@timestamp":{"order":"desc","ignore_unmapped":true}}]}'; +$search = json_decode($search); +$ts =& $search->query->filtered->filter->bool->must[0]->range->{'@timestamp'}; +$ts->from = strtotime("-$search_days days")*1000; +$ts->to = time()*1000; + +//remove timestamp limit from search +//$nothing = array_shift($search->query->filtered->filter->bool->must); + +$search->size = $fetch_results; +$search = json_encode($search); + +$date = time(); +$stats = array(); +$total_samples = 0; +while ($total_samples < $fetch_results && $date > strtotime("-$search_days days")) { + $strdate = date('Y.m.d', $date); + $date -= 86400; + + echo "Fetching results for $strdate\n"; + $url = "http://logstash.openstack.org/elasticsearch/logstash-$strdate/_search"; + $result = json_decode(fetch_results($search, $url)); + if (empty($result) || isset($result->error) || $result->status == 404) { + echo "Error fetching results for $strdate.. skipping\n"; + continue; + } + + echo "Analyzing ".count($result->hits->hits)." samples\n"; + $total_samples += process_results($result, $stats); +} + +foreach ($stats as &$branch) { + foreach ($branch as &$nodepool) { + foreach ($nodepool as &$playbook) { + $avg = array_sum($playbook) / count($playbook); + $playbook['samples'] = count($playbook); + $playbook['average'] = $avg; + } + } +} + +foreach ($stats as $bname => $branch) { + echo "Stats for openstack-ansible/$bname\n"; + foreach ($branch as $npname => $nodepool) { + echo "Node pool $npname:\n"; + uasort($nodepool, 'avg_cmp'); + foreach ($nodepool as $pbname => $playbook) { + echo "\t$pbname: ".ceil($playbook['average'])." avg, ({$playbook['samples']} samples)\n"; + } + } +} + +function fetch_results($search, $url) { + $ch = curl_init($url); + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST"); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $search); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, array( + 'Content-Type: application/json', + 'Content-Length: ' . strlen($search)) + ); + return curl_exec($ch); +} + +function process_results($result, &$stats) { + $regex = '/Operation: \[ openstack-ansible --forks [0-9]+\s+(?[^\.]+).yml \]\s+(?[0-9]+)\s+seconds/'; + foreach ($result->hits->hits as $r) { + $rs = $r->{'_source'}; + if (!preg_match($regex, $rs->message, $m)) continue; + $stats[$rs->build_branch][$rs->node_provider][$m['play_name']][] = $m['play_seconds']; + } + return count($result->hits->hits); +} + +function avg_cmp($a, $b) { + if ($a['average'] == $b['average']) { + return 0; + } + return ($a['average'] > $b['average']) ? -1 : 1; +} +?> diff --git a/osa-gate-profile/fetchlogs.php b/osa-gate-profile/fetchlogs.php new file mode 100644 index 00000000..df983f20 --- /dev/null +++ b/osa-gate-profile/fetchlogs.php @@ -0,0 +1,107 @@ +#!/usr/bin/php + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + * Argument required is json output from logstash.openstack.org's table of metrics. There is no an export option so hack this together by changing the paging size + * and watch the network traffic in chrome, then save it to a file. + * Once the file exists, feed it to this script as an argument and it will go through and download all of the log_url items to a directory called 'dump' + * You should probably mkdir dump/ in the same directory as this script first. + */ + +date_default_timezone_set('US/Central'); + +$fetch_results = 500; +$search_days = 30; + +$search = '{"query":{"filtered":{"filter":{"bool":{"must":[{"range":{"@timestamp":{"from":1463241312439,"to":1463846112439}}},{"fquery":{"query":{"query_string":{"query":"project:(\"openstack\/openstack-ansible\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"build_name:(/gate-openstack-ansible-openstack-ansible-aio-ubuntu-(trusty|xenial-nv)/)"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"message:(\"gate-check-commit.sh\")"}},"_cache":true}}]}}}},"size":'.$fetch_results.',"sort":[{"@timestamp":{"order":"desc","ignore_unmapped":true}},{"@timestamp":{"order":"desc","ignore_unmapped":true}}]}'; +$search = json_decode($search); +$ts =& $search->query->filtered->filter->bool->must[0]->range->{'@timestamp'}; +$ts->from = strtotime("-$search_days days")*1000; +$ts->to = time()*1000; + +$search->size = $fetch_results; +$search = json_encode($search); + +$date = time(); +$stats = array(); +$total_samples = 0; +while ($total_samples < $fetch_results && $date > strtotime("-$search_days days")) { + $strdate = date('Y.m.d', $date); + $date -= 86400; + + echo "Fetching results for $strdate\n"; + $url = "http://logstash.openstack.org/elasticsearch/logstash-$strdate/_search"; + $result = json_decode(fetch_results($search, $url)); + if (empty($result) || isset($result->error) || $result->status == 404) { + echo "Error fetching results for $strdate.. skipping\n"; + continue; + } + + echo "Fetching ".count($result->hits->hits)." samples\n"; + foreach($result->hits->hits as $h) { + $outfile = $h->{'_source'}->{'build_change'}.'-'.$h->{'_source'}->{'build_patchset'}.'.html'; + if (file_exists('dump/'.$outfile)) { + echo "Already downloaded $outfile..skipping\n"; + continue; + } + $url = $h->{'_source'}->{'log_url'}; + $wget = "wget -O dump/{$outfile} {$url}"; + echo "running $wget\n"; + echo "Fetching $url to $outfile\n"; + $exec = fetch_log($url, $outfile); + if ($exec['exit_code'] != 0) { + $url = $url.'.gz'; + echo "Fetch failed. Trying $url\n"; + $exec = fetch_log($url, $outfile); + if ($exec['return_code'] != 0) echo "Fetch failed retry. Skipping file.\n"; + else echo "Fetch succeeded retry\n"; + } + else echo "Fetch succeeded\n"; + } +} + +function fetch_log($url, $outfile) { + $wget = "wget -O dump/{$outfile} {$url}"; + exec($wget, $shell_output, $code); + return array('output' => $shell_output, 'exit_code' => $code); +} + +function fetch_results($search, $url) { + $ch = curl_init($url); + //curl_setopt($ch, CURLOPT_VERBOSE, true); + //curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST"); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $search); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, array( + 'Content-Type: application/json', + 'Content-Length: ' . strlen($search)) + ); + $result = curl_exec($ch); + return $result; +} + +function process_results($result, &$stats) { + $regex = '/Operation: \[ openstack-ansible --forks [0-9]+\s+(?[^\.]+).yml \]\s+(?[0-9]+)\s+seconds/'; + foreach ($result->hits->hits as $r) { + $rs = $r->{'_source'}; + if (!preg_match($regex, $rs->message, $m)) continue; + $stats[$rs->build_branch][$rs->node_provider][$m['play_name']][] = $m['play_seconds']; + } + return count($result->hits->hits); +} +?> diff --git a/osa-gate-profile/generatereports.php b/osa-gate-profile/generatereports.php new file mode 100644 index 00000000..374f6527 --- /dev/null +++ b/osa-gate-profile/generatereports.php @@ -0,0 +1,39 @@ +#!/usr/bin/php + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +$f = json_decode(file_get_contents($argv[1]), true); +$profiles = []; +$sorted = array(); +foreach ($f as $taskname => $nodeproviders) { + echo "$taskname:\n"; + foreach ($nodeproviders as $p => $n) { + $avg = round(array_sum($n) / count($n), 2); + $samples = count($n); + echo "\t$p ($samples samples) - $avg\n"; + $sorted[$p][$taskname] = $avg; + } +} + +foreach ($sorted as $k => $v) { + echo $k."\n"; + arsort($v); + foreach ($v as $tasks => $avgs) { + echo "\t$tasks = $avgs\n"; + } +} +?> diff --git a/osa-gate-profile/parselogs.php b/osa-gate-profile/parselogs.php new file mode 100644 index 00000000..c9e7bbb3 --- /dev/null +++ b/osa-gate-profile/parselogs.php @@ -0,0 +1,66 @@ +#!/usr/bin/php + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + * Parse all of the console logs saved to dump/* + * This will try to regex match the first few lines which identify the nodepool + * Then it will search through the console log for PLAY RECAP style output which contains execution times + * for all of the ansible tasks run. Once it is done it will output a json object of metrics. + * Example output at http://cdn.pasteraw.com/5efufwesy1nmxnl6qis9z1zr1hbce0 + */ +$dir = new DirectoryIterator(dirname(__FILE__).'/dump/'); +$profile = array(); +$profiles=0; + +pcntl_signal(SIGHUP, function($signo) { + global $profile; + echo json_encode($profile)."\n"; +}); +pcntl_signal(SIGTERM, function($signo) { + print_exit(); +}); +pcntl_signal(SIGINT, function($signo) { + print_exit(); +}); + +foreach ($dir as $f) { + if (!$f->isFile()) continue; + $c = file_get_contents('dump/'.$f); + if (!preg_match('/Building remotely on (?:)?([^<\s]+)-[0-9]+(?:<\/a>)?/', $c, $m)) { + file_put_contents("php://stderr", "Failed to find node type of $f\n"); + continue; + } + $nodetype = $m[1]; + foreach (explode("\n", $c) as $l) { + if (preg_match('/(?:[^|]+)\|\s+(.*)\s+[-]+\s+([0-9\.s]+)s\s*$/', $l, $m)) { + $role_task = $m[1]; + $seconds = (float)$m[2]; + $profile[$role_task][$nodetype][] = $seconds; + } + } + $profiles++; + pcntl_signal_dispatch(); +} + +print_exit(); + +function print_exit() { + global $profile, $profiles; + file_put_contents("php://stderr", "\nExiting with $profiles profiles completed.\n"); + exit(json_encode($profile)); +} + +?> diff --git a/osa-gate-profile/readme.rst b/osa-gate-profile/readme.rst new file mode 100644 index 00000000..fe38201c --- /dev/null +++ b/osa-gate-profile/readme.rst @@ -0,0 +1,27 @@ +Generate metrics from openstack-ansible gate check console logs +############################################################### +:date: 2016-10-10 +:tags: openstack, ansible +:category: \*openstack, \*nix + + +About this repository +--------------------- + +These scripts will query logstash.openstack.org to find a set of OSA gate check +console logs, download them, and perform task timing analytics. + +- step1: fetchlogs.php +- step2: parselogs.php +- step3: generatereports.php + +Example run +----------- + +.. code-block:: bash + + mkdir dump + php fetchlogs.php + php parselogs.php > intermediary.json + php generatereports.php intermediary.json + rm -rf intermediary.json