The Assimilation Project  based on Assimilation version 1.1.7.1474836767
droneinfo.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number colorcolumn=100
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Copyright (C) 2011, 2012 - Alan Robertson <alanr@unix.sh>
7 #
8 # The Assimilation software is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # The Assimilation software is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the GNU General Public License
19 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
20 #
21 #
22 '''
23 We implement the Drone class - which implements all the properties of
24 drones as a Python class.
25 '''
26 import time, sys
27 #import os, traceback
28 from cmadb import CMAdb
29 from consts import CMAconsts
30 from store import Store
31 from graphnodes import nodeconstructor, RegisterGraphClass, IPaddrNode, BPRules, \
32  NICNode
33 from systemnode import SystemNode
34 from frameinfo import FrameSetTypes, FrameTypes
35 from AssimCclasses import pyNetAddr, DEFAULT_FSP_QID, pyCryptFrame
36 from assimevent import AssimEvent
37 from cmaconfig import ConfigFile
38 from graphnodes import GraphNode
39 
40 
41 @RegisterGraphClass
42 #droneinfo.py:39: [R0904:Drone] Too many public methods (21/20)
43 #droneinfo.py:39: [R0902:Drone] Too many instance attributes (11/10)
44 # pylint: disable=R0904,R0902
45 class Drone(SystemNode):
46  '''Everything about Drones - endpoints that run our nanoprobes.
47 
48  There are two Cypher queries that get initialized later:
49  Drone.IPownerquery_1: Given an IP address, return th SystemNode (probably Drone) 'owning' it.
50  Drone.OwnedIPsQuery: Given a Drone object, return all the IPaddrNodes that it 'owns'
51  '''
52  IPownerquery_1 = None
53  OwnedIPsQuery = None
54  IPownerquery_1_txt = '''START n=node:IPaddrNode({ipaddr})
55  MATCH (n)<-[:%s]-()<-[:%s]-(drone)
56  return drone LIMIT 1'''
57  OwnedIPsQuery_txt = '''START d=node({droneid})
58  MATCH (d)-[:%s]->()-[:%s]->(ip)
59  return ip'''
60 
61 
62  # R0913: Too many arguments to __init__()
63  # pylint: disable=R0913
64  def __init__(self, designation, port=None, startaddr=None
65  , primary_ip_addr=None, domain=CMAconsts.globaldomain
66  , status= '(unknown)', reason='(initialization)', roles=None, key_id=''):
67  '''Initialization function for the Drone class.
68  We mainly initialize a few attributes from parameters as noted above...
69 
70  The first time around we also initialize a couple of class-wide query
71  strings for a few queries we know we'll need later.
72 
73  We also behave as though we're a dict from the perspective of JSON attributes.
74  These discovery strings are converted into pyConfigContext objects and are
75  then searchable like dicts themselves - however updating these dicts has
76  no direct impact on the underlying JSON strings stored in the database.
77 
78  The reason for treating these as a dict is so we can easily change
79  the implementation to put JSON strings in separate nodes, or perhaps
80  eventually in a separate data store.
81 
82  This is necessary because the performance of putting lots of really large
83  strings in Neo4j is absolutely horrible. Putting large strings in is dumb
84  and what Neo4j does with them is even dumber...
85  The result is at least DUMB^2 -not 2*DUMB ;-)
86  '''
87  SystemNode.__init__(self, domain=domain, designation=designation)
88  if roles is None:
89  roles = ['host', 'drone']
90  self.addrole(roles)
91  self._io = CMAdb.io
92  self.lastjoin = 'None'
93  self._active_nic_count = None
94  self.status = status
95  self.reason = reason
96  self.key_id = key_id
97  self.startaddr = str(startaddr)
98  self.primary_ip_addr = str(primary_ip_addr)
99  self.time_status_ms = int(round(time.time() * 1000))
100  self.time_status_iso8601 = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())
101  if port is not None:
102  self.port = int(port)
103  else:
104  self.port = None
105 
106  if Drone.IPownerquery_1 is None:
107  Drone.IPownerquery_1 = (Drone.IPownerquery_1_txt
108  % (CMAconsts.REL_ipowner, CMAconsts.REL_nicowner))
109  Drone.OwnedIPsQuery_subtxt = (Drone.OwnedIPsQuery_txt \
110  % (CMAconsts.REL_nicowner, CMAconsts.REL_ipowner))
111  Drone.OwnedIPsQuery = Drone.OwnedIPsQuery_subtxt
112  self.set_crypto_identity()
113  if Store.is_abstract(self) and not CMAdb.store.readonly:
114  #print 'Creating BP rules for', self.designation
115  from bestpractices import BestPractices
116  bprules = CMAdb.io.config['bprulesbydomain']
117  rulesetname = bprules[domain] if domain in bprules else bprules[CMAconsts.globaldomain]
118  for rule in BestPractices.gen_bp_rules_by_ruleset(CMAdb.store, rulesetname):
119  #print >> sys.stderr, 'ADDING RELATED RULE SET for', \
120  #self.designation, rule.bp_class, rule
121  CMAdb.store.relate(self, CMAconsts.REL_bprulefor, rule,
122  properties={'bp_class': rule.bp_class})
123 
125  '''Return a generator producing all the best practice rules
126  that apply to this Drone.
127  '''
128  return CMAdb.store.load_related(self, CMAconsts.REL_bprulefor, BPRules)
129 
130  def get_bp_head_rule_for(self, trigger_discovery_type):
131  '''
132  Return the head of the ruleset chain for the particular set of rules
133  that go with this particular node
134  '''
135  rules = CMAdb.store.load_related(self, CMAconsts.REL_bprulefor, BPRules)
136  for rule in rules:
137  if rule.bp_class == trigger_discovery_type:
138  return rule
139  return None
140 
141  def get_merged_bp_rules(self, trigger_discovery_type):
142  '''Return a merged version of the best practices rules for this
143  particular discovery type. This involves creating a hash table
144  of rules, where the contents are merged together such that we return
145  a single consolidated view of the rules to our viewer.
146  We start out with the head of the ruleset chain and then merge in the
147  ones its based on.
148 
149  We return a dict-like object reflecting this merger suitable
150  for evaluating the rules. You just walk the set of rules
151  and evaluate them.
152  '''
153  # Although we ought to hit the database once and get the PATH of the
154  # rules in one fell swoop, we don't yet support PATHs, so we're going
155  # at it the somewhat slower way -- incrementally.
156  start = self.get_bp_head_rule_for(trigger_discovery_type)
157  if start is None:
158  return {}
159  ret = start.jsonobj()
160  this = start
161  while True:
162  nextrule = None
163  for nextrule in CMAdb.store.load_related(this, CMAconsts.REL_basis, BPRules):
164  break
165  if nextrule is None:
166  break
167  nextobj = nextrule.jsonobj()
168  for elem in nextobj:
169  if elem not in ret:
170  ret[elem] = nextobj[elem]
171  this = nextrule
172  return ret
173 
174  @staticmethod
176  'Compute the attribute name of a best practice score category'
177  return 'bp_category_%s_score' % category
178 
179  def bp_category_list(self):
180  'Provide the list best practice score categories that we have stored'
181  result = []
182  for attr in dir(self):
183  if attr.startswith('bp_category_') and attr.endswith('_score'):
184  result.append(attr[12:-6])
185  return result
186 
188  'List the discovery types that we have recorded'
189  result = []
190  for attr in dir(self):
191  if attr.startswith('BP_') and attr.endswith('_rulestatus'):
192  result.append(attr[3:-11])
193  return result
194 
195  @staticmethod
197  'Compute the attribute name of a best practice score category'
198  return 'BP_%s_rulestatus' % discoverytype
199 
200  def get_owned_ips(self):
201  '''Return a list of all the IP addresses that this Drone owns'''
202  params = {'droneid':Store.id(self)}
203  if CMAdb.debug:
204  print >> sys.stderr, ('IP owner query:\n%s\nparams %s'
205  % (Drone.OwnedIPsQuery_subtxt, str(params)))
206 
207  return [node for node in CMAdb.store.load_cypher_nodes(Drone.OwnedIPsQuery, IPaddrNode
208  , params=params)]
209 
210  def get_owned_nics(self):
211  '''Return an iterator returning all the NICs that this Drone owns'''
212  return CMAdb.store.load_related(self, CMAconsts.REL_nicowner, NICNode)
213 
215  '''Return the number of "active" NICs this Drone has'''
216  if self._active_nic_count is not None:
217  return self._active_nic_count
218  count = 0
219  for nic in self.get_owned_nics():
220  if nic.operstate == 'up' and nic.carrier and nic.macaddr != '00-00-00-00-00-00':
221  count += 1
222  self._active_nic_count = count
223  return count
224 
225 
226  def crypto_identity(self):
227  '''Return the Crypto Identity that should be associated with this Drone
228  Note that this current algorithm isn't ideal for a multi-tenant environment.
229  '''
230  return self.designation
231 
232 
233  def destaddr(self, ring=None):
234  '''Return the "primary" IP for this host as a pyNetAddr with port'''
235  return pyNetAddr(self.select_ip(ring=ring), port=self.port)
236 
237  def select_ip(self, ring=None):
238  '''Select an appropriate IP address for talking to a partner on this ring
239  or our primary IP if ring is None'''
240  # Current code is not really good enough for the long term,
241  # but is good enough for now...
242  # In particular, when talking on a particular switch ring, or
243  # subnet ring, we want to choose an IP that's on that subnet,
244  # and preferably on that particular switch for a switch-level ring.
245  # For TheOneRing, we want their primary IP address.
246  ring = ring
247  return self.primary_ip_addr
248 
249  def send_frames(self, framesettype, frames):
250  'Send messages to our real concrete Drone system...'
251  # This doesn't work if the client has bound to a VIP
252  ourip = pyNetAddr(self.select_ip()) # meaning select our primary IP
253  if ourip.port() == 0:
254  ourip.setport(self.port)
255  #print >> sys.stderr, ('ADDING PACKET TO TRANSACTION: %s', str(frames))
256  if CMAdb.debug:
257  CMAdb.log.debug('Sending request to %s Frames: %s'
258  % (str(ourip), str(frames)))
259  CMAdb.transaction.add_packet(ourip, framesettype, frames)
260  #print >> sys.stderr, ('Sent Discovery request to %s Frames: %s'
261  #% (str(ourip), str(frames)))
262 
263 
264  #Current implementation does not use 'self'
265  #pylint: disable=R0201
266  def send_hbmsg(self, dest, fstype, addrlist):
267  '''Send a message with an attached pyNetAddr list - each including port numbers'
268  This is intended primarily for start or stop heartbeating messages.'''
269 
270  # Now we create a collection of frames that looks like this:
271  #
272  # One FrameTypes.RSCJSON frame containing JSON Heartbeat parameters
273  # one frame per dest, type FrameTypes.IPPORT
274  #
275  params = ConfigFile.agent_params(CMAdb.io.config
276  , 'heartbeats', None, self.designation)
277  framelist = [{'frametype': FrameTypes.RSCJSON, 'framevalue': str(params)},]
278  for addr in addrlist:
279  if addr is None:
280  continue
281  framelist.append({'frametype': FrameTypes.IPPORT, 'framevalue': addr})
282 
283  CMAdb.transaction.add_packet(dest, fstype, framelist)
284 
285  def death_report(self, status, reason, fromaddr, frameset):
286  'Process a death/shutdown report for us. RIP us.'
287  from hbring import HbRing
288  frameset = frameset # We don't use the frameset at this point in time
289  if reason != 'HBSHUTDOWN':
290  if self.status != status or self.reason != reason:
291  CMAdb.log.info('Node %s has been reported as %s by address %s. Reason: %s'
292  % (self.designation, status, str(fromaddr), reason))
293  oldstatus = self.status
294  self.status = status
295  self.reason = reason
296  self.monitors_activated = False
297  self.time_status_ms = int(round(time.time() * 1000))
298  self.time_status_iso8601 = time.strftime('%Y-%m-%d %H:%M:%S')
299  if status == oldstatus:
300  # He was already dead, Jim.
301  return
302  # There is a need for us to be a little more sophisticated
303  # in terms of the number of peers this particular drone had
304  # It's here in this place that we will eventually add the ability
305  # to distinguish death of a switch or subnet or site from death of a single drone
306  for mightbering in CMAdb.store.load_in_related(self, None, nodeconstructor):
307  if isinstance(mightbering, HbRing):
308  mightbering.leave(self)
309  deadip = pyNetAddr(self.select_ip(), port=self.port)
310  if CMAdb.debug:
311  CMAdb.log.debug('Closing connection to %s/%d' % (deadip, DEFAULT_FSP_QID))
312  #
313  # So, if this is a death report from another system we could shut down ungracefully
314  # and it would be OK.
315  #
316  # But if it's a graceful shutdown, we need to not screw up the comm shutdown in progress
317  # If it's broken, our tests and the real world will eventually show that up :-D.
318  #
319  if reason != 'HBSHUTDOWN':
320  self._io.closeconn(DEFAULT_FSP_QID, deadip)
321  AssimEvent(self, AssimEvent.OBJDOWN)
322 
323  def start_heartbeat(self, ring, partner1, partner2=None):
324  '''Start heartbeating to the given partners.
325  We insert ourselves between partner1 and partner2.
326  We only use forward links - because we can follow them in both directions in Neo4J.
327  So, we need to create a forward link from partner1 to us and from us to partner2 (if any)
328  '''
329  ouraddr = pyNetAddr(self.select_ip(), port=self.port)
330  partner1addr = pyNetAddr(partner1.select_ip(ring), port=partner1.port)
331  if partner2 is not None:
332  partner2addr = pyNetAddr(partner2.select_ip(ring), port=partner2.port)
333  else:
334  partner2addr = None
335  if CMAdb.debug:
336  CMAdb.log.debug('STARTING heartbeat(s) from %s [%s] to %s [%s] and %s [%s]' %
337  (self, ouraddr, partner1, partner1addr, partner2, partner2addr))
338  self.send_hbmsg(ouraddr, FrameSetTypes.SENDEXPECTHB, (partner1addr, partner2addr))
339 
340  def stop_heartbeat(self, ring, partner1, partner2=None):
341  '''Stop heartbeating to the given partners.'
342  We don't know which node is our forward link and which our back link,
343  but we need to remove them either way ;-).
344  '''
345  ouraddr = pyNetAddr(self.select_ip(), port=self.port)
346  partner1addr = pyNetAddr(partner1.select_ip(ring), port=partner1.port)
347  if partner2 is not None:
348  partner2addr = pyNetAddr(partner2.select_ip(ring), port=partner2.port)
349  else:
350  partner2addr = None
351  # Stop sending the heartbeat messages between these (former) peers
352  if CMAdb.debug:
353  CMAdb.log.debug('STOPPING heartbeat(s) from %s [%s] to %s [%s] and %s [%s]' %
354  (self, ouraddr, partner1, partner1addr, partner2, partner2addr))
355  self.send_hbmsg(ouraddr, FrameSetTypes.STOPSENDEXPECTHB, (partner1addr, partner2addr))
356 
357  def set_crypto_identity(self, keyid=None):
358  'Associate our IP addresses with our key id'
359  if CMAdb.store.readonly or not CMAdb.use_network:
360  return
361  if keyid is not None and keyid != '':
362  if self.key_id != '' and keyid != self.key_id:
363  raise ValueError('Cannot change key ids for % from %s to %s.'
364  % (str(self), self.key_id, keyid))
365  self.key_id = keyid
366  # Encryption is required elsewhere - we ignore this here...
367  if self.key_id != '':
368  pyCryptFrame.dest_set_key_id(self.destaddr(), self.key_id)
369  pyCryptFrame.associate_identity(self.crypto_identity(), self.key_id)
370 
371  def __str__(self):
372  'Give out our designation'
373  return 'Drone(%s)' % self.designation
374 
375  def find_child_system_from_json(self, jsonobj):
376  '''Locate the child drone that goes with this JSON - or maybe it's us'''
377  if 'proxy' in jsonobj:
378  path = jsonobj['proxy']
379  if path == 'local/local':
380  return self
381  else:
382  return self
383  # This works - could be a bit slow if you have lots of child nodes...
384  q = '''MATCH (drone)<-[:parentsys*]-(child)
385  WHERE ID(drone) = {id} AND child.childpath = {path}
386  RETURN child'''
387  store = Store.getstore(self)
388  child = store.load_cypher_node(q, GraphNode.factory, {'id': store.id(self), 'path': path})
389  if child is None:
390  raise(ValueError('Child system %s from %s [%s] was not found.'
391  % (path, str(self), str(Store.id(self)))))
392  return child
393 
394  @staticmethod
395  def find(designation, port=None, domain=None):
396  'Find a drone with the given designation or IP address, or Neo4J node.'
397  desigstr = str(designation)
398  if isinstance(designation, Drone):
399  designation.set_crypto_identity()
400  return designation
401  elif isinstance(designation, str):
402  if domain is None:
403  domain = CMAconsts.globaldomain
404  designation = designation.lower()
405  drone = CMAdb.store.load_or_create(Drone, port=port, domain=domain
406  , designation=designation)
407  assert drone.designation == designation
408  assert CMAdb.store.has_node(drone)
409  drone.set_crypto_identity()
410  return drone
411  elif isinstance(designation, pyNetAddr):
412  desig = designation.toIPv6()
413  desig.setport(0)
414  desigstr = str(desig)
415  if domain is None:
416  dstr = '*'
417  else:
418  dstr = domain
419  query = '%s:%s' % (str(Store.lucene_escape(desigstr)), dstr)
420  #We now do everything by IPv6 addresses...
421  drone = CMAdb.store.load_cypher_node(Drone.IPownerquery_1, Drone, {'ipaddr':query})
422  if drone is not None:
423  assert CMAdb.store.has_node(drone)
424  drone.set_crypto_identity()
425  return drone
426  if CMAdb.debug:
427  CMAdb.log.warn('Could not find IP NetAddr address in Drone.find... %s [%s] [%s]'
428  % (designation, desigstr, type(designation)))
429 
430  if CMAdb.debug:
431  CMAdb.log.debug("DESIGNATION2 (%s) = %s" % (designation, desigstr))
432  CMAdb.log.debug("QUERY (%s) = %s" % (designation, query))
433  print >> sys.stderr, ("DESIGNATION2 (%s) = %s" % (designation, desigstr))
434  print >> sys.stderr, ("QUERY (%s) = %s" % (designation, query))
435  if CMAdb.debug:
436  raise RuntimeError('drone.find(%s) (%s) (%s) => returning None' % (
437  str(designation), desigstr, type(designation)))
438  #str(designation), desigstr, type(designation)))
439  #tblist = traceback.extract_stack()
440  ##tblist = traceback.extract_tb(trace, 20)
441  #CMAdb.log.info('======== Begin missing IP Traceback ========')
442  #for tbelem in tblist:
443  #(filename, line, funcname, text) = tbelem
444  #filename = os.path.basename(filename)
445  #CMAdb.log.info('%s.%s:%s: %s'% (filename, line, funcname, text))
446  #CMAdb.log.info('======== End missing IP Traceback ========')
447  #CMAdb.log.warn('drone.find(%s) (%s) (%s) => returning None' % (
448  return None
449 
450  @staticmethod
451  def add(designation, reason, status='up', port=None, domain=CMAconsts.globaldomain
452  , primary_ip_addr=None):
453  'Add a drone to our set unless it is already there.'
454  drone = CMAdb.store.load_or_create(Drone, domain=domain, designation=designation
455  , primary_ip_addr=primary_ip_addr, port=port, status=status, reason=reason)
456  assert CMAdb.store.has_node(drone)
457  drone.reason = reason
458  drone.status = status
459  drone.statustime = int(round(time.time() * 1000))
460  drone.iso8601 = time.strftime('%Y-%m-%d %H:%M:%S')
461  if port is not None:
462  drone.port = port
463  if primary_ip_addr is not None and drone.primary_ip_addr != primary_ip_addr:
464  # This means they've changed their IP address and/or port since we last saw them...
465  CMAdb.log.info('DRONE %s changed IP address from %s to %s'
466  % (str(drone), drone.primary_ip_addr, primary_ip_addr))
467  drone.primary_ip_addr = str(primary_ip_addr)
468  if port is None:
469  drone.port = int(primary_ip_addr.port())
470  return drone
471 
def add(designation, reason, status='up', port=None, domain=CMAconsts.globaldomain, primary_ip_addr=None)
Definition: droneinfo.py:452
def start_heartbeat(self, ring, partner1, partner2=None)
Definition: droneinfo.py:323
def find_child_system_from_json(self, jsonobj)
Definition: droneinfo.py:375
def send_frames(self, framesettype, frames)
Definition: droneinfo.py:249
def bp_discoverytypes_list(self)
Definition: droneinfo.py:187
def stop_heartbeat(self, ring, partner1, partner2=None)
Definition: droneinfo.py:340
def bp_category_score_attrname(category)
Definition: droneinfo.py:175
def get_owned_nics(self)
Definition: droneinfo.py:210
def get_owned_ips(self)
Definition: droneinfo.py:200
def crypto_identity(self)
Definition: droneinfo.py:226
def find(designation, port=None, domain=None)
Definition: droneinfo.py:395
def select_ip(self, ring=None)
Definition: droneinfo.py:237
def gen_current_bp_rules(self)
Definition: droneinfo.py:124
def get_bp_head_rule_for(self, trigger_discovery_type)
Definition: droneinfo.py:130
def get_active_nic_count(self)
Definition: droneinfo.py:214
def bp_discoverytype_result_attrname(discoverytype)
Definition: droneinfo.py:196
def __str__(self)
Definition: droneinfo.py:371
def destaddr(self, ring=None)
Definition: droneinfo.py:233
def set_crypto_identity(self, keyid=None)
Definition: droneinfo.py:357
def bp_category_list(self)
Definition: droneinfo.py:179
def send_hbmsg(self, dest, fstype, addrlist)
Definition: droneinfo.py:266
def __init__(self, designation, port=None, startaddr=None, primary_ip_addr=None, domain=CMAconsts.globaldomain, status='(unknown)', reason='(initialization)', roles=None, key_id='')
Definition: droneinfo.py:66
def get_merged_bp_rules(self, trigger_discovery_type)
Definition: droneinfo.py:141
def death_report(self, status, reason, fromaddr, frameset)
Definition: droneinfo.py:285