The Assimilation Project  based on Assimilation version 1.1.7.1474836767
store_test.py
Go to the documentation of this file.
1 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number
2 #
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Copyright (C) 2011, 2012 - Alan Robertson <alanr@unix.sh>
7 #
8 # The Assimilation software is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # The Assimilation software is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the GNU General Public License
19 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
20 #
21 #
22 _suites = ['all', 'cma']
23 import sys, os
24 import gc
25 sys.path.extend(['..', '../cma', "/usr/local/lib/python2.7/dist-packages"])
26 import py2neo
27 from py2neo import neo4j, GraphError
28 from store import Store
29 from AssimCclasses import pyNetAddr, dump_c_objects
30 from AssimCtypes import ADDR_FAMILY_802, proj_class_live_object_count, proj_class_dump_live_objects
31 from graphnodes import GraphNode, RegisterGraphClass
32 from cmainit import CMAinit
33 from cmadb import Neo4jCreds
34 
35 DEBUG=False
36 CheckForDanglingClasses = True
37 AssertOnDanglingClasses = True
38 
39 WorstDanglingCount = 0
40 
41 if not CheckForDanglingClasses:
42  print >> sys.stderr, 'WARNING: Memory Leak Detection disabled.'
43 elif not AssertOnDanglingClasses:
44  print >> sys.stderr, 'WARNING: Memory Leak assertions disabled (detection still enabled).'
45 
46 print >> sys.stderr, 'USING PYTHON VERSION %s' % str(sys.version)
47 
48 version_printed = False
49 
50 def assert_no_dangling_Cclasses(doassert=None):
51  global CheckForDanglingClasses
52  global WorstDanglingCount
53  sys._clear_type_cache()
54  if doassert is None:
55  doassert = AssertOnDanglingClasses
56  CMAinit.uninit()
57  gc.collect() # For good measure...
59  #print >>sys.stderr, "CHECKING FOR DANGLING CLASSES (%d)..." % count
60  # Avoid cluttering the output up with redundant messages...
61  if count > WorstDanglingCount and CheckForDanglingClasses:
62  WorstDanglingCount = count
63  if doassert:
64  print >> sys.stderr, 'STARTING OBJECT DUMP'
65  print 'stdout STARTING OBJECT DUMP'
67  print >> sys.stderr, 'OBJECT DUMP COMPLETE'
68  print 'stdout OBJECT DUMP COMPLETE'
69  raise AssertionError("Dangling C-class objects - %d still around" % count)
70  else:
71  print >> sys.stderr, ("*****ERROR: Dangling C-class objects - %d still around" % count)
72 
73 
74 class TestCase(object):
75  def assertEqual(self, a, b):
76  assert a == b
77 
78  def assertNotEqual(self, a, b):
79  assert a != b
80 
81  def assertTrue(self, a):
82  assert a is True
83 
84  def assertFalse(self, a):
85  assert a is False
86 
87  def assertRaises(self, exception, function, *args, **kw):
88  try:
89  function(*args, **kw)
90  raise Exception('Did not raise exception %s: %s(%s)', exception, function, str(args))
91  except exception as e:
92  return True
93 
94  def teardown_method(self, method):
95  print '__del__ CALL for %s' % str(method)
97 
98 def CreateIndexes(db, indexlist):
99  'Create Indexes(indexlist) - a list of strings for Node indexes to create'
100  for index in indexlist:
101  try:
102  db.legacy.delete_index(neo4j.Node, index)
103  except GraphError:
104  pass
105  for index in indexlist:
106  db.legacy.get_or_create_index(neo4j.Node, index, None)
107 
108 
109 @RegisterGraphClass
110 class Person(GraphNode):
111  'A Person - or at least an electronic representation of one'
112  def __init__(self, firstname, lastname, dateofbirth=None):
113  GraphNode.__init__(self, domain='global')
114  self.firstname = firstname
115  self.lastname = lastname
116  if dateofbirth is not None:
117  self.dateofbirth = dateofbirth
118  else:
119  self.dateofbirth='unknown'
120 
121  @staticmethod
123  'Return our key attributes in order of significance'
124  return ['lastname', 'firstname']
125 
126 
127 @RegisterGraphClass
128 class aTestSystem(GraphNode):
129  'Some kind of semi-intelligent system'
130  def __init__(self, designation, domain='global', roles=None):
131  GraphNode.__init__(self, domain)
132  self.designation = designation.lower()
133  if roles == None:
134  roles = ['']
135  self.roles = roles
136  if domain is None:
137  domain='global'
138  self.domain = domain
139 
140  def addroles(self, role):
141  if self.roles[0] == '':
142  del self.roles[0]
143  if isinstance(role, (list, tuple)):
144  for arole in role:
145  self.addroles(arole)
146  elif role not in self.roles:
147  self.roles.append(role)
148  return self.roles
149 
150  @staticmethod
152  'Return our key attributes in order of significance'
153  return ['designation', 'domain']
154 
155 @RegisterGraphClass
157  def __init__(self, designation, domain='global', roles=None):
158  aTestSystem.__init__(self, designation=designation)
159  if roles is None:
160  roles = []
161  elif isinstance(roles, str):
162  roles = [roles]
163  roles.extend(['host', 'Drone'])
164  aTestSystem.__init__(self, designation, domain=domain, roles=roles)
165 
166 
167 @RegisterGraphClass
168 class aTestIPaddr(GraphNode):
169  def __init__(self, ipaddr):
170  GraphNode.__init__(self, domain='global')
171  if isinstance(ipaddr, str):
172  ipaddr = pyNetAddr(ipaddr)
173  if isinstance(ipaddr, pyNetAddr):
174  ipaddr = ipaddr.toIPv6()
175  self.ipaddr = str(ipaddr)
176 
177  @staticmethod
179  'Return our key attributes in order of significance'
180  return ['ipaddr']
181 
182 @RegisterGraphClass
183 class aTestNIC(GraphNode):
184  def __init__(self, MACaddr):
185  GraphNode.__init__(self, domain='global')
186  mac = pyNetAddr(MACaddr)
187  if mac is None or mac.addrtype() != ADDR_FAMILY_802:
188  raise ValueError('Not a legal MAC address [%s // %s]: %s (%s)'
189  % (MACaddr, str(mac), str(mac.addrtype()), mac.addrlen()))
190  self.MACaddr = str(mac)
191 
192  @staticmethod
194  'Return our key attributes in order of significance'
195  return ['MACaddr']
196 
197 
198 Classes = [Person, aTestSystem, aTestDrone, aTestIPaddr, aTestNIC]
199 
200 keymap = {'Person': {'index':'Person','kattr': 'lastname', 'vattr': 'firstname'},
201  'aTestSystem': {'index':'aTestSystem','kattr': 'designation', 'value': 'global'},
202  'aTestDrone': {'index':'aTestDrone','kattr': 'designation', 'value': 'global'},
203  'aTestIPaddr': {'index':'aTestIPaddr','kattr': 'ipaddr', 'value': 'global'},
204  'aTestNIC': {'index':'aTestNIC','kattr': 'MACaddr', 'value': 'global'}
205  }
206 uniqueindexes = {}
207 for key in keymap:
208  uniqueindexes[key] = True
209 
210 
211 ##mys = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
212 
223 
224 def initstore():
225  global version_printed
226  Neo4jCreds().authenticate()
227  db = neo4j.Graph(None)
228  if not version_printed:
229  print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version)
230  print >> sys.stderr, 'USING py2neo VERSION %s' % str(py2neo.__version__)
231  version_printed = True
232  GraphNode.clean_graphnodes()
233  db.delete_all()
234  CMAinit(None)
235  OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
236  OurStore.clean_store()
237  CreateIndexes(db, [cls.__name__ for cls in Classes])
238  return OurStore
239 
240 
242  def test_drone(self):
243  store = initstore()
244  seven = store.load_or_create(aTestDrone, designation='SevenOfNine', roles='Borg')
245  self.assertTrue('host' in seven.roles)
246  self.assertTrue('Drone' in seven.roles)
247  self.assertTrue('Borg' in seven.roles)
248  self.assertTrue(store.is_abstract(seven))
249  store.commit()
250  self.assertTrue('host' in seven.roles)
251  self.assertTrue('Drone' in seven.roles)
252  self.assertTrue('Borg' in seven.roles)
253  self.assertTrue(not store.is_abstract(seven))
254  self.assertTrue(isinstance(seven, aTestSystem))
255  self.assertEqual(seven.designation, 'sevenofnine')
256  self.assertFalse(store.is_abstract(seven))
257 
258  def test_person(self):
259  store = initstore()
260  #store.debug = True
261  Annika = Person('Annika', 'Hansen')
262  store.save(Annika)
263  store.commit()
264  whoami = store.load_or_create(Person, firstname='Annika', lastname='Hansen')
265  self.assertTrue(whoami is Annika)
266  self.assertEqual(Annika.firstname, 'Annika')
267  self.assertEqual(Annika.lastname, 'Hansen')
268 
269  def test_system(self):
270  store = initstore()
271  fredsys = store.load_or_create(aTestSystem, designation='Fred', roles=['bridge', 'router'])
272  self.assertTrue('bridge' in fredsys.roles)
273  self.assertTrue('router' in fredsys.roles)
274  self.assertFalse('host' in fredsys.roles)
275  self.assertFalse('drone' in fredsys.roles)
276  self.assertTrue(store.is_abstract(fredsys))
277  store.commit()
278  self.assertFalse(store.is_abstract(fredsys))
279  self.assertEqual(fredsys.designation, 'fred')
280 
281  def test_nic(self):
282  store = initstore()
283  freddiemac = store.load_or_create(aTestNIC, MACaddr='AA-BB-CC-DD-EE-FF')
284  self.assertEqual(freddiemac.MACaddr, 'aa-bb-cc-dd-ee-ff')
285  sys64 = store.load_or_create(aTestNIC, MACaddr='00-11-CC-DD-EE-FF:AA:BB')
286  self.assertEqual(sys64.MACaddr, '00-11-cc-dd-ee-ff-aa-bb')
287  self.assertRaises(ValueError, store.load_or_create, aTestNIC, MACaddr='AA-BB-CC-DD-EE-FF:')
288  self.assertRaises(ValueError, store.load_or_create, aTestNIC, MACaddr='AA-BB-CC-DD-EE-FF:00')
289  self.assertRaises(ValueError, store.load_or_create, aTestNIC, MACaddr='AA-BB-CC-DD-EE-FF-')
290  self.assertRaises(ValueError, store.load_or_create, aTestNIC, MACaddr='10.10.10.5')
291  self.assertTrue(store.is_abstract(freddiemac))
292  store.commit()
293  self.assertEqual(freddiemac.MACaddr, 'aa-bb-cc-dd-ee-ff')
294  self.assertEqual(sys64.MACaddr, '00-11-cc-dd-ee-ff-aa-bb')
295  self.assertTrue(not store.is_abstract(freddiemac))
296 
298 
299  def test_relate1(self):
300  store = initstore()
301  Annika = store.load_or_create(Person, firstname='Annika', lastname='Hansen')
302  seven = store.load_or_create(aTestDrone, designation='SevenOfNine', roles='Borg')
303  store.relate(seven, 'formerly', Annika)
304  # Borg Drones need 64 bit MAC addresses ;-) (or maybe more)
305  sevennic = store.load_or_create(aTestNIC, MACaddr='ff-ff:7-0f-9:7-0f-9')
306  store.relate(seven, 'nicowner', sevennic)
307  self.assertEqual(str(sevennic.MACaddr), 'ff-ff-07-0f-09-07-0f-09')
308  self.assertTrue(isinstance(seven, aTestSystem))
309  self.assertTrue(isinstance(seven, aTestDrone))
310  self.assertTrue(isinstance(seven, GraphNode))
311  self.assertTrue(isinstance(Annika, Person))
312  self.assertTrue(isinstance(Annika, GraphNode))
313  self.assertTrue(isinstance(sevennic, aTestNIC))
314  self.assertTrue(isinstance(sevennic, GraphNode))
315 
316  # It would be nice if the following tests would work even before committing...
317  store.commit()
318  count=0
319  for node in store.load_related(seven, 'formerly', Person):
320  self.assertTrue(node is Annika)
321  count += 1
322  self.assertEqual(count, 1)
323  count=0
324  for node in store.load_in_related(Annika, 'formerly', aTestDrone):
325  self.assertTrue(node is seven)
326  count += 1
327  self.assertEqual(count, 1)
328  count=0
329  for node in store.load_related(seven, 'nicowner', aTestNIC):
330  self.assertTrue(node is sevennic)
331  count += 1
332  self.assertEqual(count, 1)
333 
334  def test_relate2(self):
335  store = initstore()
336  seven = store.load_or_create(aTestDrone, designation='SevenOfNine', roles='Borg')
337  sevennic1 = store.load_or_create(aTestNIC, MACaddr='ff-ff:7-0f-9:7-0f-9')
338  sevennic2 = store.load_or_create(aTestNIC, MACaddr='00-00:7-0f-9:7-0f-9')
339  ipaddr1=store.load_or_create(aTestIPaddr, ipaddr='10.10.10.1')
340  ipaddr2=store.load_or_create(aTestIPaddr, ipaddr='10.10.10.2')
341  store.relate(seven, 'nicowner', sevennic1)
342  store.relate(seven, 'nicowner', sevennic2)
343  store.relate(sevennic1, 'ipowner', ipaddr1)
344  store.relate(sevennic2, 'ipowner', ipaddr2)
345  store.commit()
346 
347  prevnode = None
348  count=0
349  for node in store.load_related(seven, 'nicowner', aTestNIC):
350  self.assertTrue((node is sevennic1 or node is sevennic2) and node is not prevnode)
351  prevnode = node
352  count += 1
353  ipcount=0
354  for ip in store.load_related(node, 'ipowner', aTestIPaddr):
355  if node is sevennic1:
356  self.assertTrue(ip is ipaddr1)
357  else:
358  self.assertTrue(ip is ipaddr2)
359  ipcount += 1
360  self.assertTrue(ipcount == 1)
361  self.assertEqual(count, 2)
362 
363  def test_separate1(self):
364  store = initstore()
365  seven = store.load_or_create(aTestDrone, designation='SevenOfNine', roles='Borg')
366  sevennic1 = store.load_or_create(aTestNIC, MACaddr='ff-ff:7-0f-9:7-0f-9')
367  sevennic2 = store.load_or_create(aTestNIC, MACaddr='00-00:7-0f-9:7-0f-9')
368  ipaddr1=store.load_or_create(aTestIPaddr, ipaddr='10.10.10.1')
369  ipaddr2=store.load_or_create(aTestIPaddr, ipaddr='10.10.10.2')
370  store.relate(seven, 'nicowner', sevennic1)
371  store.relate(seven, 'nicowner', sevennic2)
372  store.relate(sevennic1, 'ipowner', ipaddr1)
373  store.relate(sevennic2, 'ipowner', ipaddr2)
374  store.commit()
375  store.separate(sevennic2, 'ipowner')
376  store.separate(seven, 'nicowner', sevennic2)
377  store.commit()
378  count=0
379  for node in store.load_related(seven, 'nicowner', aTestNIC):
380  self.assertTrue(node is sevennic1)
381  count += 1
382  ipcount=0
383  for ip in store.load_related(node, 'ipowner', aTestIPaddr):
384  self.assertTrue(ip is ipaddr1)
385  ipcount += 1
386  self.assertTrue(ipcount == 1)
387  self.assertEqual(count, 1)
388 
390 
392  store = initstore()
393  Annika = store.load_or_create(Person, firstname='Annika', lastname='Hansen')
394  seven = store.load_or_create(aTestDrone, designation='SevenOfNine', roles='Borg')
395  store.relate(seven, 'formerly', Annika)
396  sevennic1 = store.load_or_create(aTestNIC, MACaddr='ff-ff:7-0f-9:7-0f-9')
397  sevennic2 = store.load_or_create(aTestNIC, MACaddr='00-00:7-0f-9:7-0f-9')
398  ipaddr1=store.load_or_create(aTestIPaddr, ipaddr='10.10.10.1')
399  ipaddr2=store.load_or_create(aTestIPaddr, ipaddr='10.10.10.2')
400  store.relate(seven, 'nicowner', sevennic1)
401  store.relate(seven, 'nicowner', sevennic2)
402  store.relate(sevennic1, 'ipowner', ipaddr1)
403  store.relate(sevennic2, 'ipowner', ipaddr2)
404  store.commit()
405  # Now we have something to test queries against...
406  # seven-[:formerly]->Annika
407  # seven-[:nicowner]-> sevennic1-[:ipowner]->10.10.10.1
408  # seven-[:nicowner]-> sevennic2-[:ipowner]->10.10.10.2
409 
410  Qstr='''START drone=node:aTestDrone('sevenofnine:*')
411  MATCH (person)<-[:formerly]-(drone)-[:nicowner]->(nic)-[:ipowner]->(ipaddr)
412  RETURN person, drone, nic, ipaddr'''
413  iter = store.load_cypher_query(Qstr, GraphNode.factory)
414  rowcount = 0
415  foundaddr1=False
416  foundaddr2=False
417  for row in iter:
418  rowcount += 1
419  # fields are person, drone, nic and ipaddr
420  self.assertTrue(row.person is Annika)
421  self.assertTrue(row.drone is seven)
422  if row.nic is sevennic1:
423  self.assertTrue(row.ipaddr is ipaddr1)
424  foundaddr1 = True
425  else:
426  self.assertTrue(row.nic is sevennic2)
427  self.assertTrue(row.ipaddr is ipaddr2)
428  foundaddr2 = True
429  self.assertEqual(rowcount, 2)
430  self.assertTrue(foundaddr1)
431  self.assertTrue(foundaddr2)
432 
434  mac1= 'ff-ff:7-0f-9:7-0f-9'
435  mac2= '00-00:7-0f-9:7-0f-9'
436  ip1= '10.10.10.1'
437  ip2= '10.10.10.2'
438 
439  def create_stuff(self, store):
440  seven = store.load_or_create(aTestDrone, designation='SevenOfNine', roles='Borg')
441  #Annika = store.load_or_create(Person, firstname='Annika', lastname='Hansen')
442  #store.relate(seven, 'formerly', Annika)
443  #sevennic1 = store.load_or_create(aTestNIC, MACaddr=TestDatabaseWrites.mac1)
444  #sevennic2 = store.load_or_create(aTestNIC, MACaddr=TestDatabaseWrites.mac2)
445  #ipaddr1=store.load_or_create(aTestIPaddr, ipaddr=TestDatabaseWrites.ip1)
446  #ipaddr2=store.load_or_create(aTestIPaddr, ipaddr=TestDatabaseWrites.ip2)
447  #store.relate(seven, 'nicowner', sevennic1)
448  #store.relate(seven, 'nicowner', sevennic2)
449  #store.relate(sevennic1, 'ipowner', ipaddr1)
450  #store.relate(sevennic2, 'ipowner', ipaddr2)
451  store.commit()
452  # Now we have something to test queries against...
453  # seven-[:formerly]->Annika
454  # seven-[:nicowner]-> sevennic1-[:ipowner]->10.10.10.1
455  # seven-[:nicowner]-> sevennic2-[:ipowner]->10.10.10.2
456  # When we return, we should not have anything in our cache
457 
459  '''The main point of this test is to verify that things actually go into the
460  database correctly.'''
461  Qstr='''START drone=node:aTestDrone('sevenofnine:*')
462  MATCH (person)<-[:formerly]-(drone)-[:nicowner]->(nic)-[:ipowner]->(ipaddr)
463  RETURN person, drone, nic, ipaddr'''
464  Qstr='''START drone=node:aTestDrone('sevenofnine:*')
465  RETURN drone'''
466  store = initstore()
467  #print >> sys.stderr, 'RUNNING create_stuff'
468  self.create_stuff(store) # Everything has gone out of scope
469  # so nothing is cached any more
470  #print >> sys.stderr, 'RUNNING test_create_and_query'
471  # Verify nothing is cached any more
472  self.assertEqual(store.batchindex, 0)
473  self.assertEqual(len(store.clients), 0)
474  self.assertEqual(len(store.newrels), 0)
475  self.assertEqual(len(store.deletions), 0)
476  self.assertEqual(len(store.deletions), 0)
477  gc.collect()
478  danglingweakref=False
479  if False:
480  # This subtest used to work, but once I added AssimEvents to the mix
481  # some of our former objects now hang around - I have no idea why...
482  # This varies by OS and python version - but not in any rational way...
483  for ref in store.weaknoderefs:
484  wref = store.weaknoderefs[ref]()
485  if wref is not None:
486  print >> sys.stderr, ('OOPS: weakref %s still alive' % str(wref))
487  print >> sys.stderr, ('PYTHON VERSION: %s' % str(sys.version))
488  danglingweakref = True
489  self.assertTrue(not danglingweakref)
490  store.weaknoderefs = {}
491  iter = store.load_cypher_query(Qstr, GraphNode.factory)
492  rowcount = 0
493  foundaddr1=False
494  foundaddr2=False
495  for row in iter:
496  rowcount += 1
497  # fields are person, drone, nic and ipaddr
498  #self.assertEqual(row.person.firstname, 'Annika')
499  #self.assertEqual(row.person.lastname, 'Hansen')
500  self.assertEqual(row.drone.designation, 'SevenOfNine'.lower())
501  for role in ('host', 'Drone', 'Borg'):
502  self.assertTrue(role in row.drone.roles)
503  if False and row.nic.MACaddr == TestDatabaseWrites.mac1:
504  self.assertEqual(row.ipaddr.ipaddr, TestDatabaseWrites.ip1)
505  foundaddr1 = True
506  elif False:
507  self.assertEqual(row.nic.MACaddr, TestDatabaseWrites.mac2)
508  self.assertEqual(row.ipaddr.ipaddr, TestDatabaseWrites.ip2)
509  foundaddr2 = True
510  self.assertEqual(rowcount, 1)
511  #self.assertTrue(foundaddr1)
512  #self.assertTrue(foundaddr2)
513 
514 # Other things that ought to have tests:
515 # node deletion
516 # Searching for nodes we just added (I forgot which ones work that way)
517 # Filtered queries - note that fields have to be filtered out in the JSON
518 # they can't be reliably filtered out of the nodes
519 # other things?
520 
521 if __name__ == "__main__":
522  run()
523 
def initstore()
mys = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
Definition: store_test.py:224
def assertNotEqual(self, a, b)
Definition: store_test.py:78
def __init__(self, designation, domain='global', roles=None)
Definition: store_test.py:157
guint32 proj_class_live_object_count(void)
Return the count of live C class objects.
Definition: proj_classes.c:406
def CreateIndexes(db, indexlist)
Definition: store_test.py:98
def assertRaises(self, exception, function, args, kw)
Definition: store_test.py:87
def assertEqual(self, a, b)
Definition: store_test.py:75
def __init__(self, MACaddr)
Definition: store_test.py:184
def __init__(self, firstname, lastname, dateofbirth=None)
Definition: store_test.py:112
def teardown_method(self, method)
Definition: store_test.py:94
def assert_no_dangling_Cclasses(doassert=None)
Definition: store_test.py:50
def __init__(self, designation, domain='global', roles=None)
Definition: store_test.py:130