The Assimilation Project  based on Assimilation version 1.1.7.1474836767
systemnode.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number colorcolumn=100
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Copyright (C) 2011, 2012 - Alan Robertson <alanr@unix.sh>
7 #
8 # The Assimilation software is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # The Assimilation software is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the GNU General Public License
19 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
20 #
21 #
22 ''' This module defines the classes for several of our System nodes ... '''
23 import sys, time
24 from consts import CMAconsts
25 from store import Store
26 from cmadb import CMAdb
27 from AssimCclasses import pyConfigContext
28 from graphnodes import RegisterGraphClass, GraphNode, JSONMapNode, \
29  add_an_array_item, delete_an_array_item, nodeconstructor
30 from cmaconfig import ConfigFile
31 from AssimCtypes import CONFIGNAME_TYPE
32 from frameinfo import FrameTypes, FrameSetTypes
33 
34 @RegisterGraphClass
35 class SystemNode(GraphNode):
36  'An object that represents a physical or virtual system (server, switch, etc)'
37  HASH_PREFIX = 'JSON__hash__'
38  JSONattrnames = '''START d=node({droneid})
39  MATCH (d)-[r:jsonattr]->()
40  return r.jsonname as key'''
41  JSONsingleattr = '''START d=node({droneid})
42  MATCH (d)-[r:jsonattr]->(json)
43  WHERE r.jsonname={jsonname}
44  return json'''
45 
46  _JSONprocessors = None # This will get updated
47 
48  def __init__(self, domain, designation, roles=None):
49  GraphNode.__init__(self, domain=domain)
50  self.designation = str(designation).lower()
51  self.monitors_activated = False
52  if roles is None or roles == []:
53  # Neo4j can't initialize node properties to empty arrays because
54  # it wants to know what kind of array it is...
55  roles = ['']
56  self.roles = roles
57 
58  @staticmethod
60  'Return our key attributes in order of significance'
61  return ['designation', 'domain']
62 
63  def addrole(self, roles):
64  'Add a role to our SystemNode'
65  self.roles = add_an_array_item(self.roles, roles)
66  # Make sure the 'roles' attribute gets marked as dirty...
67  Store.mark_dirty(self, 'roles')
68  return self.roles
69 
70  def delrole(self, roles):
71  'Delete a role from our SystemNode'
72  self.roles = delete_an_array_item(self.roles, roles)
73  Store.mark_dirty(self, 'roles')
74  return self.roles
75 
76  def logjson(self, origaddr, jsontext):
77  'Process and save away JSON discovery data.'
78  assert CMAdb.store.has_node(self)
79  jsonobj = pyConfigContext(jsontext)
80  if 'instance' not in jsonobj or 'data' not in jsonobj:
81  CMAdb.log.warning('Invalid JSON discovery packet: %s' % jsontext)
82  return
83  dtype = jsonobj['instance']
84  discoverychanged = not self.json_eq(dtype, jsontext)
85  if discoverychanged:
86  CMAdb.log.debug("Saved discovery type %s [%s] for endpoint %s."
87  % (jsonobj['discovertype'], dtype, self.designation))
88  else:
89  if CMAdb.debug:
90  CMAdb.log.debug('Discovery type %s for endpoint %s is unchanged.'
91  % (dtype, self.designation))
92  self._process_json(origaddr, jsonobj, discoverychanged)
93  self[dtype] = jsontext # This is stored in separate nodes for performance
94 
95 
96  def __iter__(self):
97  'Iterate over our child JSON attribute names'
98  for tup in CMAdb.store.load_cypher_query(self.JSONattrnames, None,
99  params={'droneid': Store.id(self)}):
100  yield str(tup.key)
101 
102  def keys(self):
103  'Return the names of all our JSON discovery attributes.'
104  return [str(attr) for attr in self]
105 
106  def __contains__(self, key):
107  'Return True if our object contains the given key (JSON name).'
108  return hasattr(self, str(self.HASH_PREFIX + key))
109 
110  def __len__(self):
111  'Return the number of JSON items in this SystemNode.'
112  return len(self.keys())
113 
114  def jsonval(self, jsontype):
115  'Construct a python object associated with a particular JSON discovery value.'
116  if not hasattr(self, str(self.HASH_PREFIX + jsontype)):
117  #print >> sys.stderr, 'DOES NOT HAVE ATTR %s' % jsontype
118  #print >> sys.stderr, 'ATTRIBUTES ARE:' , str(self.keys())
119  return None
120  #print >> sys.stderr, 'LOADING', self.JSONsingleattr, \
121  # {'droneid': Store.id(self), 'jsonname': jsontype}
122  node = CMAdb.store.load_cypher_node(self.JSONsingleattr, JSONMapNode,
123  params={'droneid': Store.id(self),
124  'jsonname': str(jsontype)}
125  )
126  #assert self.json_eq(jsontype, str(node))
127  return node
128 
129  def get(self, key, alternative=None):
130  '''Return JSON object if the given key exists - 'alternative' if not.'''
131  ret = self.deepget(str(key))
132  return ret if ret is not None else alternative
133 
134  def __getitem__(self, key):
135  'Return the given JSON value or raise IndexError.'
136  ret = self.jsonval(key)
137  if ret is None:
138  raise IndexError('No such JSON attribute [%s].' % key)
139  return ret
140 
141  def deepget(self, key, alternative=None):
142  '''Return value if object contains the given *structured* key - 'alternative' if not.'''
143  keyparts = key.split('.', 1)
144  if len(keyparts) == 1:
145  ret = self.jsonval(key)
146  return ret if ret is not None else alternative
147  jsonmap = self.jsonval(keyparts[0])
148  return alternative if jsonmap is None else jsonmap.deepget(keyparts[1], alternative)
149 
150  def __setitem__(self, name, value):
151  'Set the given JSON value to the given object/string.'
152  if name in self:
153  if self.json_eq(name, value):
154  return
155  else:
156  #print >> sys.stderr, 'DELETING ATTRIBUTE', name
157  # FIXME: ADD ATTRIBUTE HISTORY (enhancement)
158  # This will likely involve *not* doing a 'del' here
159  del self[name]
160  jsonnode = CMAdb.store.load_or_create(JSONMapNode, json=value)
161  setattr(self, self.HASH_PREFIX + name, jsonnode.jhash)
162  CMAdb.store.relate(self, CMAconsts.REL_jsonattr, jsonnode,
163  properties={'jsonname': name,
164  'time': long(round(time.time()))
165  })
166 
167  def __delitem__(self, name):
168  'Delete the given JSON value from the SystemNode.'
169  #print >> sys.stderr, 'ATTRIBUTE DELETION:', name
170  jsonnode=self.get(name, None)
171  try:
172  delattr(self, self.HASH_PREFIX + name)
173  except AttributeError:
174  raise IndexError('No such JSON attribute [%s].' % name)
175  if jsonnode is None:
176  CMAdb.log.warning('Missing JSON attribute: %s' % name)
177  print >> sys.stderr, ('Missing JSON attribute: %s' % name)
178  return
179  should_delnode = True
180  # See if it has any remaining references...
181  for node in CMAdb.store.load_in_related(jsonnode,
182  CMAconsts.REL_jsonattr,
183  nodeconstructor):
184  if node is not self:
185  should_delnode = False
186  break
187  CMAdb.store.separate(self, CMAconsts.REL_jsonattr, jsonnode)
188  if should_delnode:
189  # Avoid dangling properties...
190 
191  CMAdb.log.warning('Deleting old attribute value: %s [%s]' % (name, str(jsonnode)))
192  CMAdb.store.delete(jsonnode)
193 
194  def json_eq(self, key, newvalue):
195  '''Return True if this new value is equal to the current value for
196  the given key (JSON attribute name).
197 
198  We do this by comparing hash values. This keeps us from having to
199  fetch potentially very large strings (read VERY SLOW) if we can
200  compare hash values instead.
201 
202  Our hash values are representable in fewer than 60 bytes to maximize Neo4j performance.
203  '''
204  if key not in self:
205  return False
206  hashname = self.HASH_PREFIX + key
207  oldhash = getattr(self, hashname)
208  newhash = JSONMapNode.strhash(str(pyConfigContext(newvalue)))
209  #print >> sys.stderr, 'COMPARING %s to %s for value %s' % (oldhash, newhash, key)
210  return oldhash == newhash
211 
212  def send_frames(self, _framesettype, _frames):
213  'Send messages to our parent - or their parent, or their parent...'
214  raise ValueError('Cannot send frames to a %s - must be a subclass'
215  % (str(self.__class__)))
216 
217  def request_discovery(self, args): ##< A vector of arguments containing
218  '''Send our System a request to perform discovery
219  We send a DISCNAME frame with the instance name
220  then an optional DISCINTERVAL frame with the repeat interval
221  then a DISCJSON frame with the JSON data for the discovery operation.
222 
223  Our argument is a vector of pyConfigContext objects with values for
224  'instance' Name of this discovery instance
225  'interval' How often to repeat this discovery action
226  'timeout' How long to wait before considering this discovery failed...
227  '''
228  #fs = pyFrameSet(FrameSetTypes.DODISCOVER)
229  frames = []
230  for arg in args:
231  agent_params = ConfigFile.agent_params(CMAdb.io.config, 'discovery',
232  arg[CONFIGNAME_TYPE], self.designation)
233  for key in ('repeat', 'warn' 'timeout', 'nice'):
234  if key in agent_params and key not in arg:
235  arg[key] = agent_params[arg]
236  instance = arg['instance']
237  frames.append({'frametype': FrameTypes.DISCNAME, 'framevalue': instance})
238  if 'repeat' in arg:
239  interval = int(arg['repeat'])
240  frames.append({'frametype': FrameTypes.DISCINTERVAL, 'framevalue': int(interval)})
241  else:
242  interval = 0
243  frames.append({'frametype': FrameTypes.DISCJSON, 'framevalue': str(arg)})
244  self.send_frames(FrameSetTypes.DODISCOVER, frames)
245 
246 
247  def _process_json(self, origaddr, jsonobj, discoverychanged):
248  'Pass the JSON data along to interested discovery plugins (if any)'
249  dtype = jsonobj['discovertype']
250  foundone = False
251  if CMAdb.debug:
252  CMAdb.log.debug('Processing JSON for discovery type [%s]' % dtype)
253  for prio in range(0, len(SystemNode._JSONprocessors)):
254  if dtype in SystemNode._JSONprocessors[prio]:
255  foundone = True
256  classes = SystemNode._JSONprocessors[prio][dtype]
257  #print >> sys.stderr, 'PROC[%s][%s] = %s' % (prio, dtype, str(classes))
258  for cls in classes:
259  proc = cls(CMAdb.io.config, CMAdb.transaction, CMAdb.store
260  , CMAdb.log, CMAdb.debug)
261  proc.processpkt(self, origaddr, jsonobj, discoverychanged)
262  if foundone:
263  CMAdb.log.info('Processed %schanged %s JSON data from %s into graph.'
264  % ('' if discoverychanged else 'un', dtype, self.designation))
265  elif discoverychanged:
266  CMAdb.log.info('Stored %s JSON data from %s without processing.'
267  % (dtype, self.designation))
268 
269  @staticmethod
270  def add_json_processor(clstoadd):
271  "Register (add) all the json processors we've been given as arguments"
272 
273  if SystemNode._JSONprocessors is None:
274  SystemNode._JSONprocessors = []
275  for _prio in range(0, clstoadd.PRI_LIMIT):
276  SystemNode._JSONprocessors.append({})
277 
278  priority = clstoadd.priority()
279  msgtypes = clstoadd.desiredpackets()
280 
281  for msgtype in msgtypes:
282  if msgtype not in SystemNode._JSONprocessors[priority]:
283  SystemNode._JSONprocessors[priority][msgtype] = []
284  if clstoadd not in SystemNode._JSONprocessors[priority][msgtype]:
285  SystemNode._JSONprocessors[priority][msgtype].append(clstoadd)
286 
287  return clstoadd
288 
289 @RegisterGraphClass
291  'A class representing a Child System (like a VM or a container)'
292 
293  DiscoveryPath = None
294 
295  # pylint R0913: too many arguments - needed because of the way we retrieve from the database
296  # and we never call the constructor directly - we call it via "childfactory" or the
297  # database calls it with args
298  # pylint: disable=R0913
299  def __init__(self, designation, _parentsystem=None, domain=None, roles=None, _selfjson=None,
300  uniqueid=None, childpath=None):
301  #print >> sys.stderr, 'CONSTRUCTING CHILD NODE!====================: %s' % str(designation)
302  if domain is None:
303  domain=_parentsystem.domain
304  SystemNode.__init__(self, domain=domain, designation=designation, roles=roles)
305  self._selfjson = _selfjson
306  if uniqueid is None:
307  uniqueid = ChildSystem.compute_uniqueid(designation, _parentsystem, domain)
308  self.uniqueid = uniqueid
309  if childpath is None:
310  if hasattr(_parentsystem, 'childpath'):
311  childpath = '%s/%s:%s' % (self.__class__.DiscoveryPath, designation,
312  _parentsystem.childpath)
313  else:
314  childpath = '%s/%s' % (self.__class__.DiscoveryPath, designation)
315  self.childpath = childpath
316  self.runas_user = None
317  self.runas_group = None
318  if _parentsystem is not None:
319  self._parentsystem = _parentsystem
320  #print >> sys.stderr, 'YAY GOT A CHILD NODE!=====================: %s' % str(self)
321 
322  def post_db_init(self):
323  '''Do post-constructor database updates'''
324  if not hasattr(self, '_parentsystem'):
325  for node in CMAdb.store.load_related(self, CMAconsts.REL_parentsys, nodeconstructor):
326  self._parentsystem = node
327  break
328  if self._selfjson is not None:
329  self['selfjson'] = self._selfjson
330  if not hasattr(self, '_parentsystem'):
331  raise RuntimeError('Cannot find parent system for %s (%s)' % (type(self), self))
332  if self._parentsystem.__class__ is SystemNode:
333  raise ValueError('Parent system cannot be a base "SystemNode" object')
334 
335  def send_frames(self, framesettype, frames):
336  'Send messages to our parent - or their parent, or their parent...'
337  self._parentsystem.send_frames(framesettype, frames)
338 
339 
340  @staticmethod
341  def compute_uniqueid(designation, parentsystem, domain=None):
342  'We compute the unique id we use to find this in the database'
343  if domain is None:
344  domain = parentsystem.domain
345  if hasattr(parentsystem, 'uniqueid'):
346  return getattr(parentsystem, 'uniqueid') + '::' + designation
347  return ('%s::%s::%s' %(designation, parentsystem.designation, domain))
348 
349  @staticmethod
350  def childfactory(parentsystem, childtype, designation, jsonobj, roles=None, domain=None):
351  'We construct an appropriate ChildSystem subclass object - or find it in the database'
352  store = Store.getstore(parentsystem)
353  if childtype == 'docker':
354  cls = DockerSystem
355  elif childtype == 'vagrant':
356  cls = VagrantSystem
357  else:
358  raise ValueError('Unknown ChildSystem type(%s)' % childtype)
359  uniqueid = ChildSystem.compute_uniqueid(designation, parentsystem, domain)
360  return store.load_or_create(cls, designation=designation, _selfjson=str(jsonobj),
361  _parentsystem=parentsystem, roles=roles, uniqueid=uniqueid)
362 
363  @staticmethod
365  'Return our key attributes in order of significance'
366  return ['uniqueid']
367 
368 @RegisterGraphClass
370  'A class representing a Docker container'
371  DiscoveryPath='docker'
372  def post_db_init(self):
373  ChildSystem.post_db_init(self)
374  self.runas_user = 'nobody'
375  self.runas_group = 'docker'
376 
377 @RegisterGraphClass
379  'A class representing a Vagrant VM'
380  DiscoveryPath='vagrant'
381 
382  def post_db_init(self):
383  ChildSystem.post_db_init(self)
384  if not hasattr(self, 'runas_user'):
385  jsonobj = pyConfigContext(self._selfjson)
386  self.runas_user = jsonobj['user']
387  self.runas_group = jsonobj['group']
388 
389 if __name__ == '__main__':
390  def maintest():
391  'test main program'
392  return 0
393 
394  sys.exit(maintest())
def __contains__(self, key)
Definition: systemnode.py:106
def jsonval(self, jsontype)
Definition: systemnode.py:114
def compute_uniqueid(designation, parentsystem, domain=None)
Definition: systemnode.py:341
def logjson(self, origaddr, jsontext)
Definition: systemnode.py:76
def delrole(self, roles)
Definition: systemnode.py:70
def __init__(self, domain, designation, roles=None)
Definition: systemnode.py:48
def addrole(self, roles)
Definition: systemnode.py:63
def add_json_processor(clstoadd)
Definition: systemnode.py:270
def __getitem__(self, key)
Definition: systemnode.py:134
def childfactory(parentsystem, childtype, designation, jsonobj, roles=None, domain=None)
Definition: systemnode.py:350
def get(self, key, alternative=None)
Definition: systemnode.py:129
def add_an_array_item(currarray, itemtoadd)
Definition: graphnodes.py:246
def __setitem__(self, name, value)
Definition: systemnode.py:150
def deepget(self, key, alternative=None)
Definition: systemnode.py:141
def delete_an_array_item(currarray, itemtodel)
Definition: graphnodes.py:261
def send_frames(self, _framesettype, _frames)
Definition: systemnode.py:212
def json_eq(self, key, newvalue)
Definition: systemnode.py:194
def request_discovery(self, args)
Definition: systemnode.py:217
def send_frames(self, framesettype, frames)
Definition: systemnode.py:335
def _process_json(self, origaddr, jsonobj, discoverychanged)
Definition: systemnode.py:247
def __init__(self, designation, _parentsystem=None, domain=None, roles=None, _selfjson=None, uniqueid=None, childpath=None)
Definition: systemnode.py:300
def __delitem__(self, name)
Definition: systemnode.py:167