The Assimilation Project  based on Assimilation version 1.1.7.1474836767
logwatcher.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Author: Alan Robertson <alanr@unix.sh>
7 # Copyright (C) 2014 - Assimilation Systems Limited
8 #
9 # This file derived from CTS.py in the Linux-HA project.
10 # That file written and copyrighted (2000, 2001) by Alan Robertson.
11 # The owner contributed it to the Assimilation Project [and Assimilation Systems Limited]
12 # in August 2014.
13 # That original file was created and copyrighted before he joined IBM and was excluded
14 # from his IP agreement with IBM. The first version was written before he
15 # joined SuSE as it was the earliest component of the CTS testing system for Linux-HA.
16 #
17 # Free support is available from the Assimilation Project community - http://assimproj.org
18 # Paid support is available from Assimilation Systems Limited - http://assimilationsystems.com
19 #
20 # The Assimilation software is free software: you can redistribute it and/or modify
21 # it under the terms of the GNU General Public License as published by
22 # the Free Software Foundation, either version 3 of the License, or
23 # (at your option) any later version.
24 #
25 # The Assimilation software is distributed in the hope that it will be useful,
26 # but WITHOUT ANY WARRANTY; without even the implied warranty of
27 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 # GNU General Public License for more details.
29 #
30 # You should have received a copy of the GNU General Public License
31 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
32 #
33 #
34 '''
35 This module defines the most excellent LogWatcher class - useful for watching logs
36 and curing what ails you ;-)
37 '''
38 import os, re, time
39 
40 # R0902: too many instance attributes
41 # pylint: disable=R0902
42 class LogWatcher(object):
43 
44  '''This class watches logs for messages that fit certain regular
45  expressions. Watching logs for events isn't the ideal way
46  to do business, but it's better than nothing :-)
47 
48  On the other hand, this class is really pretty cool ;-)
49 
50  The way you use this class is as follows:
51  Construct a LogWatcher object
52  Call setwatch() when you want to start watching the log
53  Call look() to scan the log looking for the patterns
54  or call lookforall() to locate them all
55  '''
56 
57  def __init__(self, log, regexes, timeout=10, debug=None, returnonlymatch=False):
58  '''This is the constructor for the LogWatcher class. It takes a
59  log name to watch, and a list of regular expressions to watch for."
60 
61  @FIXME: should store the compiled regexes - no point in waiting around
62  until later.
63  '''
64 
65  self.patterns=None
66  self.regexes=None
67  self.setregexes(regexes)
68  self.filename = log
69  self.debug=debug
70  self.returnonlymatch = returnonlymatch
71  self.whichmatched = -1
72  self.unmatched = None
73  self.st_ino = None
74  self.st_dev = None
75  self.logfile = None
76  self.size = None
77  self.debugmsg("Debug now on for for log: %s" % log)
78  self.Timeout = int(timeout)
79  if not os.access(log, os.R_OK):
80  raise ValueError("File [" + log + "] not accessible (r)")
81 
82  def setregexes(self, regexes):
83  '''Set or modify the collection of regular expressions that we're looking for'''
84  self.patterns = []
85  self.regexes = []
86  # Validate our arguments. Better sooner than later ;-)
87  for regex in regexes:
88  self.patterns.append(re.compile(regex))
89  self.regexes.append(regex)
90 
91  def setwatch(self, frombeginning=False):
92  '''Mark the place to start watching the log from.
93  '''
94  if self.logfile is not None:
95  self.logfile.close()
96  self.logfile = None
97  self.logfile = open(self.filename, "r")
98  self.size = os.path.getsize(self.filename)
99  fsinfo = os.stat(self.filename)
100  self.st_dev = fsinfo.st_dev
101  self.st_ino = fsinfo.st_ino
102  if not frombeginning:
103  self.logfile.seek(0,2)
104 
105  def ReturnOnlyMatch(self, onlymatch=True):
106  '''Set the ReturnOnlyMatch flag
107  '''
108  self.returnonlymatch = onlymatch
109 
110  def debugmsg(self, msg, level=1):
111  'Print out a debug message if requested debugging level is activated'
112  if self.debug >= level:
113  #os.system("logger -s '%s'" % msg)
114  print 'DEBUG: %s' % msg
115 
116  # FIXME: many branches (R0912)? -- got it down to 15. Could do better...
117  # pylint: disable=R0912
118  def look(self, timeout=None):
119  '''Examine the log looking for the given patterns.
120  It starts looking from the place marked by setwatch().
121  This function looks in the file in the fashion of tail -f.
122  It properly recovers from log file truncation, and it should
123  recover from removing and recreating the log as well.
124 
125  We return the first line which matches any of our patterns.
126  '''
127  last_line=None
128  first_line=None
129  if timeout is None:
130  timeout = self.Timeout
131 
132  done=time.time()+timeout+1
133  self.debugmsg("starting search: timeout=%d" % timeout)
134  self.debugmsg("Looking for regex: %s" % str(self.regexes))
135 
136  while (timeout <= 0 or time.time() <= done):
137  newsize=os.path.getsize(self.filename)
138  self.debugmsg("newsize = %d" % newsize, 5)
139  if newsize < self.size:
140  # Somebody truncated the log!
141  self.debugmsg("Log truncated!")
142  self.setwatch(frombeginning=True)
143  continue
144  if newsize > self.logfile.tell():
145  line=self.logfile.readline()
146  self.debugmsg("Looking at line: %s" % line, 2)
147  if line:
148  last_line=line
149  if not first_line:
150  first_line=line
151  self.debugmsg("First line: "+ line)
152  match = self._match_against_regexes(line)
153  if match is not None:
154  return match
155  else: # make sure the file hasn't been recreated...
156  fsinfo = os.stat(self.filename)
157  if fsinfo.st_dev != self.st_dev or fsinfo.st_ino != self.st_ino:
158  self.debugmsg("Log file %s recreated!" % self.filename)
159  self.setwatch(frombeginning=True)
160 
161  newsize=os.path.getsize(self.filename)
162  if self.logfile.tell() == newsize:
163  if timeout > 0:
164  time.sleep(0.025)
165  else:
166  self.debugmsg("End of file")
167  self.debugmsg("Last line: %s " % str(last_line))
168  self.unmatched = self.regexes
169  return None
170  self.debugmsg("Timeout")
171  self.debugmsg("Last line: %s " % str(last_line))
172  self.unmatched = self.regexes
173  return None
174 
175  def _match_against_regexes(self, line):
176  'Match this line against all our regexes - return appropriate value'
177  which=-1
178  for regex in self.regexes:
179  which += 1
180  self.debugmsg("Comparing line to " + regex, 4)
181  #matchobj = re.search(string.lower(regex), string.lower(line))
182  matchobj = re.search(regex, line)
183  if matchobj:
184  self.whichmatched=which
185  if self.returnonlymatch:
186  return matchobj.group(self.returnonlymatch)
187  else:
188  self.debugmsg("Returning line: " + line)
189  return line
190  return None
191 
192  def lookforall(self, timeout=None, logloadavg=True):
193  '''Examine the log looking for ALL of the given patterns.
194  It starts looking from the place marked by setwatch().
195 
196  We return when the timeout is reached, or when we have found
197  ALL of the regexes that were part of the watch
198  Note that the order of the REGEXes is not relevant. They can
199  be occur in the logs in any order. Hope that's what you wanted ;-)
200  '''
201 
202  if timeout is None:
203  timeout = self.Timeout
204  save_regexes = self.regexes
205  returnresult = []
206  while (len(self.regexes) > 0):
207  oneresult = self.look(timeout)
208  if not oneresult:
209  self.unmatched = self.regexes
210  self.regexes = save_regexes
211  return None
212  returnresult.append(oneresult)
213  self.debugmsg('Deleting %s' % str(self.regexes[self.whichmatched]))
214  del self.regexes[self.whichmatched]
215  if logloadavg:
216  os.system('logger "Load Avg: $(cat /proc/loadavg)"')
217  self.unmatched = None
218  self.regexes = save_regexes
219  return returnresult
220 
221 # In case we ever want multiple regexes to match a single line...
222 #- del self.regexes[self.whichmatched]
223 #+ tmp_regexes = self.regexes
224 #+ self.regexes = []
225 #+ which = 0
226 #+ for regex in tmp_regexes:
227 #+ matchobj = re.search(regex, oneresult)
228 #+ if not matchobj:
229 #+ self.regexes.append(regex)
230 if __name__ == "__main__":
231 
232  def logmessage(msg):
233  'Log a message to our system log'
234  os.system("logger 'LOGWATCHER TEST: %s'" % msg)
235 
236  def testlog(logfilename):
237  'Function for doing basic testing of LogWatcher module'
238  watcher = LogWatcher(logfilename
239  , ['LOGWATCHER.*(Test message 1)', 'LOGWATCHER.*(Test message 2)']
240  , debug=0, returnonlymatch=True, timeout=2)
241  watcher.setwatch()
242  logmessage('Test message 2')
243  result = watcher.look()
244  assert(result == 'Test message 2')
245 
246  watcher.setwatch()
247  logmessage('Test message 1')
248  result = watcher.look()
249  assert(result == 'Test message 1')
250 
251  watcher.setwatch()
252  logmessage('Test message 2')
253  logmessage('Test message 1')
254  results = watcher.lookforall()
255  assert results == ['Test message 2', 'Test message 1']
256 
257  # Test the no-match case
258  watcher.setwatch()
259  result = watcher.look()
260  assert result is None
261 
262  print 'No asserts: Test succeded! WOOT!!'
263 
264  if os.access('/var/log/syslog', os.R_OK):
265  testlog('/var/log/syslog')
266  elif os.access('/var/log/messages', os.R_OK):
267  testlog('/var/log/messages')
def __init__(self, log, regexes, timeout=10, debug=None, returnonlymatch=False)
Definition: logwatcher.py:57
def testlog(logfilename)
Definition: logwatcher.py:236
def ReturnOnlyMatch(self, onlymatch=True)
Definition: logwatcher.py:105
def lookforall(self, timeout=None, logloadavg=True)
Definition: logwatcher.py:192
def logmessage(msg)
Definition: logwatcher.py:232
def debugmsg(self, msg, level=1)
Definition: logwatcher.py:110
def setwatch(self, frombeginning=False)
Definition: logwatcher.py:91
def look(self, timeout=None)
Definition: logwatcher.py:118
def _match_against_regexes(self, line)
Definition: logwatcher.py:175
def setregexes(self, regexes)
Definition: logwatcher.py:82