commit 338baee8d01efce080fd752c310c8e3700924798
author Paul Hobbs <phobbs@google.com> Wed Jul 13 13:42:34 2016 -0700
committer Paul Hobbs <phobbs@google.com> Thu Jul 14 17:12:17 2016 +0000
tree 51c7a8182bc457d92f9eb04bda953517ba0248a7
parent ee5a8c3e7156974b924c8bd5f3e0786ea3896749

apache_log_metrics: Use stdin.readline

sys.stdin does not begin reading lines until a large stdin buffer has
been filled or the stdin filehandle has been closed. This affects the
latency of the apache_log_metrics script - instead, we want it to begin
emitting metrics as soon as an input line is available.

Furthermore, tail_until_writer_finished buffers its output to
sys.stdout, which causes the same latency problem. Solve this by passing
"-u" flag to python, forcing unbuffered output (and input).

TEST=ran script locally, unittests pass.
BUG=chromium:621745

Change-Id: Ie882d5c3a6d939145287c306e3833672a7abc572
Reviewed-on: https://chromium-review.googlesource.com/360332
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Tested-by: Paul Hobbs <phobbs@google.com>

apache_log_metrics.py

diff --git a/apache_log_metrics.py b/apache_log_metrics.py
index bfcc6ae..9545fe0 100755
--- a/apache_log_metrics.py
+++ b/apache_log_metrics.py
@@ -12,7 +12,6 @@
 from __future__ import print_function
 
 import argparse
-import logging
 import re
 import sys
 
@@ -20,6 +19,7 @@
 
 from chromite.lib import ts_mon_config
 from chromite.lib import metrics
+from chromite.lib import cros_logging as logging
 from infra_libs import ts_mon
 
 
@@ -79,8 +79,10 @@
 
 def RunMatchers(stream, matchers):
   """Parses lines of |stream| using patterns and emitters from |matchers|"""
-  for line in stream:
+  for line in iter(stream.readline, ''):
     for matcher, emitter in matchers:
+      logging.debug('Emitting %s for input "%s"',
+                    emitter.__name__, line.strip())
       m = matcher.match(line)
       if m:
         emitter(m)