The Assimilation Project
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
tests/cma_test.py
Go to the documentation of this file.
1 # vim: smartindent tabstop=4 shiftwidth=4 expandtab
2 #
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Copyright (C) 2011, 2012 - Alan Robertson <alanr@unix.sh>
7 #
8 # The Assimilation software is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # The Assimilation software is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the GNU General Public License
19 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
20 #
21 #
22 _suites = ['all', 'cma']
23 import sys
24 sys.path.append("../pyclasswrappers")
25 sys.path.append("../cma")
26 sys.path.append("/usr/local/lib/python2.7/dist-packages")
27 from testify import *
28 from testify.utils import turtle
29 
30 from frameinfo import *
31 from AssimCclasses import *
32 import gc, sys, time, collections, os
33 from cma import *
34 
35 
36 WorstDanglingCount = 0
37 CheckForDanglingClasses = True
38 DEBUG=False
39 DoAudit=True
40 SavePackets=True
41 doHBDEAD=False
42 MaxDrone=4
43 MaxDrone=3
44 MaxDrone=10000
45 MaxDrone=5
46 BuildListOnly = True
47 
48 t1 = MaxDrone
49 if t1 < 1000: t1 = 1000
50 t2 = MaxDrone/100
51 if t2 < 10: t2 = 10
52 t3 = t2
53 
54 if BuildListOnly:
55  doHBDEAD=False
56  SavePackets=False
57  DoAudit=False
58  CheckForDanglingClasses=False
59  DEBUG=False
60 
61 
62 
63 #gc.set_threshold(t1, t2, t3)
64 
66  global CheckForDanglingClasses
67  global WorstDanglingCount
68  CMAdb.cdb = None
69  CMAdb.io = None
70  CMAdb.TheOneRing = None
71  HbRing.ringnames = {}
72  gc.collect() # For good measure...
74  #print >>sys.stderr, "CHECKING FOR DANGLING CLASSES (%d)..." % count
75  # Avoid cluttering the output up with redundant messages...
76  if count > WorstDanglingCount and CheckForDanglingClasses:
77  WorstDanglingCount = count
79  raise AssertionError, "Dangling C-class objects - %d still around" % count
80 
81 # Values to substitute into this string via '%' operator:
82 # dronedesignation (%s) MAC address byte (%02x), MAC address byte (%02x), IP address (%s)
83 netdiscoveryformat='''
84 {
85  "discovertype": "netconfig",
86  "description": "IP Network Configuration",
87  "source": "netconfig",
88  "host": "%s",
89  "data": {
90  "eth0": {
91  "address": "00:1b:fc:1b:%02x:%02x",
92  "carrier": 1,
93  "duplex": "full",
94  "mtu": 1500,
95  "operstate": "up",
96  "speed": 1000,
97  "default_gw": true,
98  "ipaddrs": { "%s/16": {"brd":"10.20.255.255", "scope":"global", "name":"eth0"}}
99  },
100  "lo": {
101  "address": "00:00:00:00:00:00",
102  "carrier": 1,
103  "mtu": 16436,
104  "operstate": "unknown",
105  "ipaddrs": { "127.0.0.1/8": {"scope":"host"}, "::1/128": {"scope":"host"}}
106  }
107  }
108 }
109 '''
110 
111 
112 byte1 = 10
113 byte2 = 20
114 
115 def droneipaddress(hostnumber):
116  byte2 = int(hostnumber / 65536)
117  byte3 = int((hostnumber / 256) % 256)
118  byte4 = hostnumber % 256
119  return pyNetAddr([byte1,byte2,byte3,byte4],)
120 
121 def dronedesignation(hostnumber):
122  return 'drone%06d' % hostnumber
123 
124 def hostdiscoveryinfo(hostnumber):
125  byte3 = int(hostnumber / 256)
126  byte4 = hostnumber % 256
127  s = str(droneipaddress(hostnumber))
128  return netdiscoveryformat % (dronedesignation(hostnumber), byte3, byte4, s)
129 
130 def geninitconfig(ouraddr):
131  return {
132  'cmainit': ouraddr, # Initial 'hello' address
133  'cmaaddr': ouraddr, # not sure what this one does...
134  'cmadisc': ouraddr, # Discovery packets sent here
135  'cmafail': ouraddr, # Failure packets sent here
136  'cmaport': 1984,
137  'hbport': 1984,
138  'outsig': pySignFrame(1),
139  'deadtime': 10*1000000,
140  'warntime': 3*1000000,
141  'hbtime': 1*1000000,
142  }
143 
144 class AUDITS(TestCase):
145  def auditadrone(self, droneid):
146  designation = dronedesignation(droneid)
147  droneip = droneipaddress(droneid)
148  droneipstr = str(droneip)
149  # Did the drone get put in the DroneInfo table?
150  drone=DroneInfo.find(designation)
151  self.assertTrue(drone is not None)
152  # Did the drone's list of addresses get updated?
153  ipnodes = drone.node.get_related_nodes('incoming', 'iphost')
154  self.assertEqual(len(ipnodes), 1)
155  ipnode = ipnodes[0]
156  ipnodeaddr = ipnode['name']
157  ipnodeaddrfoo = ipnodeaddr + '/16'
158  json = drone['JSON_netconfig']
159  jsobj = pyConfigContext(init=json)
160  jsdata = jsobj['data']
161  eth0obj = jsdata['eth0']
162  # Does the drone address table match the info from JSON?
163  eth0addrs = eth0obj['ipaddrs']
164  self.assertTrue(eth0addrs.has_key(ipnodeaddrfoo))
165  # Do we know that eth0 is the default gateway?
166  self.assertEqual(eth0obj['default_gw'], 1)
167 
168  # the JSON should have exactly 5 top-level keys
169  self.assertEqual(len(jsobj.keys()), 5)
170  # Was the JSON host name saved away correctly?
171  self.assertEqual(jsobj['host'], designation)
172 
173  return
174  peercount=0
175  ringcount=0
176  for ring in drone.ringmemberships.values():
177  ringcount += 1
178  # How many peers should it have?
179  if len(ring.memberlist) == 1:
180  pass # No peers in this ring...
181  elif len(ring.memberlist) == 2:
182  peercount += 1
183  else:
184  peercount += 2
185  # Make sure we're listed under our designation
186  #print >>sys.stderr, "DRONE is %s status %s" % (drone.designation, drone.status)
187  #print >>sys.stderr, "DRONE ringmemberships:", drone.ringmemberships.keys()
188  self.assertEqual(ring.members[drone.designation].designation, drone.designation)
189  self.assertEqual(len(ring.members), len(ring.memberlist))
190  if drone.status != 'dead':
191  # We have to be members of at least one ring...
192  self.assertTrue(ringcount >= 1)
193  # Drone should be a member of one ring (for now)
194  self.assertEqual(len(drone.ringmemberships),1)
195  # Do we have the right number of ring peers?
196  #print >>sys.stderr, "Checking peer count for drone %s (%d)" % (drone, len(drone.ringpeers))
197  self.assertEqual(len(drone.ringpeers), peercount)
198 
199  def auditSETCONFIG(self, packetreturn, droneid, configinit):
200  toaddr = packetreturn[0]
201  sentfs = packetreturn[1]
202  droneip = droneipaddress(droneid)
203 
204  # Was it a SETCONFIG packet?
205  self.assertEqual(sentfs.get_framesettype(), FrameSetTypes.SETCONFIG)
206  # Was the SETCONFIG sent back to the drone?
207  self.assertEqual(toaddr,droneip)
208  # Lets check the number of Frames in the SETCONFIG Frameset
209  configlen = len(configinit)-1 # We do not send Frames in configinfo
210  expectedlen = 2 * configlen + 4 # each address has a port that goes with it
211  self.assertEqual(expectedlen, len(sentfs)) # Was it the right size?
212 
213  def auditaRing(self, ringname):
214  'Verify that each ring has its neighbor pairs set up properly'
215  # Check that each element of the ring is connected to its neighbors...
216  ring = HbRing.ringnames[ringname]
217  #print "Ring %s: %s" % (ringname, str(ring))
218  listmembers = {}
219  ringmembers = {}
220  for drone in ring.members():
221  ringmembers[drone.node['name']] = None
222  for drone in ring.membersfromlist():
223  listmembers[drone.node['name']] = None
224  for drone in listmembers.keys():
225  self.assertTrue(drone in ringmembers)
226  for drone in ringmembers.keys():
227  self.assertTrue(drone in listmembers)
228 
229 
231  audit = AUDITS()
232  dronetype = CMAdb.cdb.nodetypetbl['Drone']
233  droneobjs = dronetype.get_related_nodes('incoming', 'IS_A')
234  numdrones = len(droneobjs)
235  for droneid in range(0,numdrones):
236  audit.auditadrone(droneid+1)
237 
239  audit = AUDITS()
240  for ring in HbRing.ringnames:
241  audit.auditaRing(ring)
242 
243 class TestIO:
244  '''A pyNetIOudp replacement for testing. It is given a list of packets to be 'read' and in turn
245  saves all the packets it 'writes' for us to inspect.
246  '''
247  def __init__(self, addrframesetpairs, sleepatend=0):
248  if isinstance(addrframesetpairs, tuple):
249  addrframesetpairs = addrframesetpairs
250  self.inframes = addrframesetpairs
252  self.packetsread=0
253  self.sleepatend=sleepatend
254  self.index=0
255  self.writecount=0
256  self.config = {CONFIGNAME_CMAPORT: 1984}
257 
258  def recvframesets(self):
259  # Audit after each packet is processed - and once before the first packet.
260  if DoAudit:
261  if self.packetsread < 200 or (self.packetsread % 500) == 0:
263  auditallrings()
264  if self.index >= len(self.inframes):
265  time.sleep(self.sleepatend)
266  raise StopIteration('End of Packets')
267  ret = self.inframes[self.index]
268  self.index += 1
269  self.packetsread += len(ret[1])
270  return ret
271 
272  def sendframesets(self, dest, fslist):
273  if not isinstance(fslist, collections.Sequence):
274  return self._sendaframeset(dest, fslist)
275  for fs in fslist:
276  self._sendaframeset(dest, fs)
277 
278  def _sendaframeset(self, dest, fslist):
279  self.writecount += 1
280  if SavePackets:
281  self.packetswritten.append((dest,fslist))
282 
283  def getmaxpktsize(self): return 60000
284  def getfd(self): return 4
285  def bindaddr(self, addr): return
286  def setblockio(self, tf): return
287 
288  def dumppackets(self):
289  print >>sys.stderr, 'Sent %d packets' % len(self.packetswritten)
290  for packet in self.packetswritten:
291  print '%s (%s)' % (packet[0], packet[1])
292 
293 
294 class TestTestInfrastructure(TestCase):
295  def test_eof(self):
296  'Get EOF with empty input'
297  if BuildListOnly: return
298  framesets=[]
299  io = TestIO(framesets, 0)
300  CMAdb.initglobal(io, True)
301  # just make sure it seems to do the right thing
302  self.assertRaises(StopIteration, io.recvframesets)
304 
305  def test_get1pkt(self):
306  'Read a single packet'
307  if BuildListOnly: return
308  otherguy = pyNetAddr([1,2,3,4],)
309  strframe1=pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.")
310  fs = pyFrameSet(42)
311  fs.append(strframe1)
312  framesets=((otherguy, (strframe1,)),)
313  io = TestIO(framesets, 0)
314  CMAdb.initglobal(io, True)
315  gottenfs = io.recvframesets()
316  self.assertEqual(len(gottenfs), 2)
317  self.assertEqual(gottenfs, framesets[0])
318  self.assertRaises(StopIteration, io.recvframesets)
319 
320  def test_echo1pkt(self):
321  'Read a packet and write it back out'
322  if BuildListOnly: return
323  strframe1=pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.")
324  fs = pyFrameSet(42)
325  fs.append(strframe1)
326  otherguy = pyNetAddr([1,2,3,4],)
327  framesets=((otherguy, (strframe1,)),)
328  io = TestIO(framesets, 0)
329  CMAdb.initglobal(io, True)
330  fslist = io.recvframesets() # read in a packet
331  self.assertEqual(len(fslist), 2)
332  self.assertEqual(fslist, framesets[0])
333  io.sendframesets(fslist[0], fslist[1]) # echo it back out
334  self.assertEqual(len(io.packetswritten), 1)
335  self.assertEqual(len(io.packetswritten), len(framesets))
336  self.assertRaises(StopIteration, io.recvframesets)
337 
338  @class_teardown
339  def tearDown(self):
341 
342 class TestCMABasic(TestCase):
343  def test_startup(self):
344  '''A semi-interesting test: We send a STARTUP message and get back a
345  SETCONFIG message with lots of good stuff in it.'''
346  if BuildListOnly: return
347  droneid = 1
348  droneip = droneipaddress(droneid)
349  designation = dronedesignation(droneid)
350  designationframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
351  dronediscovery=hostdiscoveryinfo(droneid)
352  discoveryframe=pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery)
353  fs = pyFrameSet(FrameSetTypes.STARTUP)
354  fs.append(designationframe)
355  fs.append(discoveryframe)
356  fsin = ((droneip, (fs,)),)
357  io = TestIO(fsin,0)
358  CMAdb.initglobal(io, True)
359  OurAddr = pyNetAddr((10,10,10,200),1984)
360  disp = MessageDispatcher({FrameSetTypes.STARTUP: DispatchSTARTUP()})
361  configinit = geninitconfig(OurAddr)
362  config = pyConfigContext(init=configinit)
363  listener = PacketListener(config, disp, io=io)
364  # We send the CMA an intial STARTUP packet
365  self.assertRaises(StopIteration, listener.listen) # We audit after each packet is processed
366  # Let's see what happened...
367 
368  self.assertEqual(len(io.packetswritten), 2) # Did we send out two packets?
369  # Note that this change over time
370  # As we change discovery...
371  AUDITS().auditSETCONFIG(io.packetswritten[0], droneid, configinit)
372  # Drone and Ring tables are automatically audited after each packet
373 
375  '''A very interesting test: We send a STARTUP message and get back a
376  SETCONFIG message and then send back a bunch of discovery requests.'''
377  OurAddr = pyNetAddr((10,10,10,200), 1984)
378  configinit = geninitconfig(OurAddr)
379  fsin = []
380  droneid=0
381  for droneid in range(1,MaxDrone+1):
382  droneip = droneipaddress(droneid)
383  designation = dronedesignation(droneid)
384  designationframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
385  dronediscovery=hostdiscoveryinfo(droneid)
386  discoveryframe=pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery)
387  fs = pyFrameSet(FrameSetTypes.STARTUP)
388  fs.append(designationframe)
389  fs.append(discoveryframe)
390  fsin.append((droneip, (fs,)))
391  addrone = droneipaddress(1)
392  maxdrones = droneid
393  if doHBDEAD:
394  for droneid in range(2,maxdrones+1):
395  droneip = droneipaddress(droneid)
396  deadframe=pyAddrFrame(FrameTypes.IPADDR, addrstring=droneip)
397  fs = pyFrameSet(FrameSetTypes.HBDEAD)
398  fs.append(deadframe)
399  fsin.append((addrone, (fs,)))
400  io = TestIO(fsin)
401  CMAdb.initglobal(io, True)
402  disp = MessageDispatcher( {
403  FrameSetTypes.STARTUP: DispatchSTARTUP(),
404  FrameSetTypes.HBDEAD: DispatchHBDEAD(),
405  })
406  config = pyConfigContext(init=configinit)
407  listener = PacketListener(config, disp, io=io)
408  # We send the CMA a BUNCH of intial STARTUP packets
409  try:
410  listener.listen()
411  except StopIteration as foo:
412  pass
413  #self.assertRaises(StopIteration, listener.listen)
414  # We audit after each packet is processed
415  # The auditing code will make sure all is well...
416  # But it doesn't know how many drones we just registered
417  droneroot = CMAdb.cdb.nodetypetbl['Drone']
418  Dronerels = droneroot.get_relationships('incoming', 'IS_A')
419  self.assertEqual(len(Dronerels), maxdrones)
420  if doHBDEAD:
421  partnercount = 0
422  livecount = 0
423  ringcount = 0
424  for dronerel in Dronerels:
425  drone1 = DroneInfo(dronerel.start_node)
426  if drone1.node['status'] != 'dead': livecount += 1
427  drone1rels = drone1.node.get_relationships()
428  for rel in drone1rels:
429  reltype = rel.type
430  if reltype.startswith(HbRing.memberprefix):
431  ringcount += 1
432  if reltype.startswith(HbRing.nextprefix):
433  partnercount += 1
434  self.assertEqual(partnercount, 0)
435  self.assertEqual(livecount, 1)
436  self.assertEqual(ringcount, 1)
437  if DoAudit:
439  auditallrings()
440 
441  print "The CMA read %d packets." % io.packetsread
442  print "The CMA wrote %d packets." % io.writecount
443  #io.dumppackets()
444 
445 
446  @class_teardown
447  def tearDown(self):
449 
450 if __name__ == "__main__":
451  run()