The Assimilation Project  based on Assimilation version 1.1.7.1474836767
assimtest.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Author: Alan Robertson <alanr@unix.sh>
7 # Copyright (C) 2014 - Assimilation Systems Limited
8 #
9 # Free support is available from the Assimilation Project community - http://assimproj.org
10 # Paid support is available from Assimilation Systems Limited - http://assimilationsystems.com
11 #
12 # The Assimilation software is free software: you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation, either version 3 of the License, or
15 # (at your option) any later version.
16 #
17 # The Assimilation software is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
24 #
25 #
26 '''
27 This is the main program for our Assimilation System Test module.
28 '''
29 import os, sys, random, datetime, optparse, struct
30 sys.path.append('..')
31 from logwatcher import LogWatcher
32 from testcases import AssimSysTest
33 
34 def logit(msg):
35  'log things to the system log and to stdout'
36  os.system("logger '%s'" % (msg))
37  print ("%s: %s" % (datetime.datetime.now(), msg))
38 
39 def perform_tests(testset, sysenv, store, itermax, logname, debug=False):
40  'Actually perform the given set of tests the given number of times, etc'
41  badregexes=(r' (ERROR:|CRIT:|CRITICAL:|nanoprobe\[[0-9]*]: segfault at|'
42  #r'Peer at address .* is dead|'
43  r'OUTALLDONE .* while in state NONE'
44  r')',)
45  itercount=1
46  while True:
47  test = random.choice(testset)
48  badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
49  logit("STARTING test %d - %s" % (itercount, test.__name__))
50  os.system('logger -s "Load Avg: $(cat /proc/loadavg)"')
51  os.system('logger -s "$(grep MemFree: /proc/meminfo)"')
52  badwatch.setwatch()
53  if test.__name__ == 'DiscoverService':
54  testobj = test(store, logname, sysenv, debug=debug
55  , service='bind9', monitorname='named')
56  else:
57  testobj = test(store, logname, sysenv, debug=debug)
58  ret = testobj.run()
59  match = badwatch.look()
60  if match is not None:
61  logit('BAD MESSAGE from Test %d %s: %s' % (itercount, test.__name__, match))
62  testobj.replace_result(AssimSysTest.FAIL)
63  ret = AssimSysTest.FAIL
64  if ret == AssimSysTest.SUCCESS:
65  logit('Test %d %s succeeded!' % (itercount, test.__name__))
66  itercount += 1
67  elif ret == AssimSysTest.FAIL:
68  logit('Test %d %s FAILED :-(' % (itercount, test.__name__))
69  itercount += 1
70  elif ret == AssimSysTest.SKIPPED:
71  logit('Test %d %s skipped' % (itercount, test.__name__))
72  else:
73  logit('Test %d %s RETURNED SOMETHING REALLY WEIRD [%s]'
74  % (itercount, test.__name__, str(ret)))
75  testobj.replace_result(AssimSysTest.FAIL)
76  print ''
77  if itercount > itermax:
78  break
79  return summarize_tests()
80 
82  '''Summarize the results of the tests - to syslog and stderr
83  We return the number of failures.
84  '''
85  testnames = AssimSysTest.testnames.keys()
86  testnames.sort()
87  maxlen = max([len(name) for name in testnames])
88  totals = {AssimSysTest.SUCCESS:0, AssimSysTest.FAIL:0, AssimSysTest.SKIPPED:0}
89  logit('%*s %7s %7s %7s' % (maxlen, 'TEST NAME', 'SUCCESS', 'FAIL', 'SKIPPED'))
90  for name in testnames:
91  logit('%*s %7d %7d %7d' % (maxlen, name
92  , AssimSysTest.stats[name][AssimSysTest.SUCCESS]
93  , AssimSysTest.stats[name][AssimSysTest.FAIL]
94  , AssimSysTest.stats[name][AssimSysTest.SKIPPED]))
95  totals[AssimSysTest.SUCCESS] += AssimSysTest.stats[name][AssimSysTest.SUCCESS]
96  totals[AssimSysTest.FAIL] += AssimSysTest.stats[name][AssimSysTest.FAIL]
97  totals[AssimSysTest.SKIPPED] += AssimSysTest.stats[name][AssimSysTest.SKIPPED]
98  logit('%*s %7s %7s %7s' % (maxlen, '_' * maxlen, '_____', '_____', '_____'))
99  logit('%*s %7d %7d %7d' % (maxlen, 'TOTALS'
100  , totals[AssimSysTest.SUCCESS]
101  , totals[AssimSysTest.FAIL]
102  , totals[AssimSysTest.SKIPPED]))
103  if totals[AssimSysTest.FAIL] == 0:
104  logit('%*s' % (maxlen, 'ALL TESTS SUCCEEDED!'))
105  else:
106  logit('%*s' % (maxlen, ('%d TESTS FAILED :-(' % totals[AssimSysTest.FAIL])))
107  return totals[AssimSysTest.FAIL]
108 
109 def testmain(logname):
110  'This is the actual main program for the assimilation tests'
111  maxdrones=10
112  itercount=30
113  testset = []
114 
115  parser = optparse.OptionParser(prog='assimtest'
116  , description='System Test utility for the Assimilation software.'
117  , usage=
118  '''assimtest.py [options] iteration-count [number-of-systems-in-test-environment]
119  The number of systems defaults to iteration-count/4.
120  The minimum number of nanoprobe-only systems will always be >= 2.
121  You must run this as root and have docker.io installed.''')
122  parser.add_option('-t', '--testcases'
123  , action='append'
124  , default=[]
125  , dest='testcases'
126  , choices=AssimSysTest.testnames.keys()
127  , help='specific test cases to use')
128 
129  parser.add_option('-c', '--cmadebug'
130  , action='store'
131  , default=0
132  , dest='cmadebug'
133  , choices=('0', '1', '2', '3', '4', '5')
134  , help='CMA debug level [0-5]')
135 
136  parser.add_option('-n', '--nanodebug'
137  , action='store'
138  , default=0
139  , dest='nanodebug'
140  , choices=('0', '1', '2', '3', '4', '5')
141  , help='Nanoprobe debug level [0-5]')
142 
143  parser.add_option('-s', '--seed'
144  , action='store'
145  , default=None
146  , dest='seed'
147  , help
148  = 'Random seed - a comma-separated list of 8 integers between 0 and 255 - from previous run')
149 
150  (opts, args) = parser.parse_args()
151  opts.cmadebug = int(opts.cmadebug)
152  opts.nanodebug = int(opts.nanodebug)
153 
154  if len(args) == 1:
155  itercount = int(args[0])
156  maxdrones = max(int((itercount+2) / 4), 2)
157  elif len(args) == 2:
158  itercount = int(args[0])
159  maxdrones = int(args[1])
160  else:
161  parser.parse_args(['--help'])
162  return 1
163 
164  if opts.seed is None:
165  # Prepare Random number generator
166  f = open("/dev/urandom", "r")
167  seed=struct.unpack("BBBBBBBB", f.read(8))
168  f.close()
169  else:
170  seed = tuple([int(elem) for elem in opts.seed.split(',')])
171  assert len(seed) == 8
172  random.Random().seed(seed) # The hash of those 8 bytes is used as the seed
173 
174  print '\n'
175  logit('Iteration count: %d / Number of client systems: %d' % (itercount, maxdrones))
176  logit("Random Seed: %s" % str(seed))
177 
178  print '\n'
179 
180  # Set up specific test cases -- if requested
181  if len(opts.testcases) > 0:
182  testset = [AssimSysTest.testnames[name] for name in opts.testcases]
183  else:
184  testset = [test for test in AssimSysTest.testset]
185 
186  # Set up the test environment as requested
187  env, store = AssimSysTest.initenviron(logname, maxdrones
188  , (opts.cmadebug > 0 or opts.nanodebug > 0)
189  , cmadebug=opts.cmadebug, nanodebug=opts.nanodebug)
190 
191  logit('CMA: %s %15s %6d %s' % (env.cma.hostname, env.cma.ipaddr, env.cma.pid, env.cma.name))
192  for nano in env.nanoprobes:
193  logit('nano: %s %15s %6d %s' % (nano.hostname, nano.ipaddr, nano.pid, nano.name))
194 
195  print '\n'
196  logit('STARTING %d tests on %d nanoprobes + CMA' % (itercount, maxdrones))
197  return perform_tests(testset, env, store, itercount, logname)
198 
199 
200 sys.stdout = sys.stderr # Get rid of that nasty buffering...
201 sys.exit(testmain('/var/log/syslog'))
202 
def summarize_tests()
Definition: assimtest.py:81
def logit(msg)
Definition: assimtest.py:34
def testmain(logname)
Definition: assimtest.py:109
def perform_tests(testset, sysenv, store, itermax, logname, debug=False)
Definition: assimtest.py:39