The Assimilation Project  based on Assimilation version 1.1.7.1474836767
graphnodeexpression.py
Go to the documentation of this file.
1 
2 #!/usr/bin/env python
3 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number colorcolumn=100
4 #
5 # This file is part of the Assimilation Project.
6 #
7 # Copyright (C) 2013, 2014 - Alan Robertson <alanr@unix.sh>
8 #
9 # The Assimilation software is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # The Assimilation software is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
21 #
22 #
23 ''' This module defines Functions to evaluate GraphNode expressions... '''
24 
25 import re, os, inspect, sys
26 from AssimCtypes import ADDR_FAMILY_IPV4, ADDR_FAMILY_IPV6
27 from AssimCclasses import pyNetAddr, pyConfigContext
28 #
29 #
30 class GraphNodeExpression(object):
31  '''We implement Graph node expressions - we are't a real class'''
32  functions = {}
33  def __init__(self):
34  raise NotImplementedError('This is not a real class')
35 
36  @staticmethod
37  def evaluate(expression, context):
38  '''
39  Evaluate an expression.
40  It can be:
41  None - return None
42  'some-value -- return some-value (it's a constant)
43  or an expression to find in values or graphnodes
44  or @functionname(args) - for defined functions...
45 
46  We may add other kinds of expressions in the future...
47  '''
48  if not isinstance(expression, (str, unicode)):
49  # print >> sys.stderr, 'RETURNING NONSTRING:', expression
50  return expression
51  expression = str(expression.strip())
52  if not hasattr(context, 'get') or not hasattr(context, '__setitem__'):
53  context = ExpressionContext(context)
54  # print >> sys.stderr, '''EVALUATE('%s') (%s):''' % (expression, type(expression))
55  # The value of this parameter is a constant...
56  if expression.startswith('"'):
57  if expression[-1] != '"':
58  print >> sys.stderr, "Unterminated string '%s'" % expression
59  # print >> sys.stderr, '''Constant string: "%s"''' % (expression[1:-1])
60  return expression[1:-1] if expression[-1] == '"' else None
61  if (expression.startswith('0x') or expression.startswith('0X')) and len(expression) > 3:
62  return int(expression[2:], 16)
63  if expression.isdigit():
64  return int(expression, 8) if expression.startswith('0') else int(expression)
65  if expression.find('(') >= 0:
66  value = GraphNodeExpression.functioncall(expression, context)
67  if isinstance(value, unicode):
68  value = str(value)
69  context[expression] = value
70  return value
71  # if expression.startswith('$'):
72  # print >> sys.stderr, 'RETURNING VALUE OF %s' % expression[1:]
73  # print >> sys.stderr, 'Context is %s' % str(context)
74  # print >> sys.stderr, 'RETURNING VALUE OF %s = %s'\
75  # % (expression, context.get(expression[1:], None))
76  value = context.get(expression[1:], None) if expression.startswith('$') else expression
77  if isinstance(value, unicode):
78  value = str(value)
79  return value
80 
81  # pylint R0912: too many branches - really ought to write a lexical analyzer and parser
82  # On the whole it would be simpler and easier to understand...
83  # pylint: disable=R0912
84  @staticmethod
85  def _compute_function_args(arglist, context):
86  '''Compute the arguments to a function call. May contain function calls
87  and other GraphNodeExpression, or quoted strings...
88  Ugly lexical analysis.
89  Really ought to write a real recursive descent parser...
90  '''
91  # print >> sys.stderr, '_compute_function_args(%s)' % str(arglist)
92  args = []
93  argstrings = []
94  nestcount=0
95  arg = ''
96  instring = False
97  prevwasquoted = False
98  for char in arglist:
99  if instring:
100  if char == '"':
101  instring = False
102  prevwasquoted = True
103  else:
104  arg += char
105  elif nestcount == 0 and char == '"':
106  instring = True
107  elif nestcount == 0 and char == ',':
108  if prevwasquoted:
109  prevwasquoted = False
110  args.append(arg)
111  argstrings.append(arg)
112  else:
113  arg = arg.strip()
114  if arg == '':
115  continue
116  #print >> sys.stderr, "EVALUATING [%s]" % arg
117  args.append(GraphNodeExpression.evaluate(arg, context))
118  argstrings.append(arg)
119  arg = ''
120  elif char == '(':
121  nestcount += 1
122  #print >> sys.stderr, "++nesting: %d" % (nestcount)
123  arg += char
124  elif char == ')':
125  arg += char
126  nestcount -= 1
127  #print >> sys.stderr, "--nesting: %d" % (nestcount)
128  if nestcount < 0:
129  return (None, None)
130  if nestcount == 0:
131  if prevwasquoted:
132  #print >> sys.stderr, '_compute_function_args: QUOTED argument: "%s"' % arg
133  args.append(arg)
134  else:
135  arg = arg.strip()
136  #print >> sys.stderr, "GnE.functioncall [%s]" % arg
137  args.append(GraphNodeExpression.functioncall(arg, context))
138  argstrings.append(arg)
139  arg = ''
140  else:
141  arg += char
142  if nestcount > 0 or instring:
143  #print "Nestcount: %d, instring: %s" % (nestcount, instring)
144  return (None, None)
145  if arg != '':
146  if prevwasquoted:
147  #print >> sys.stderr, '_compute_function_args: quoted argument: "%s"' % arg
148  args.append(arg)
149  else:
150  #print >> sys.stderr, "GnE.evaluate [%s]" % arg
151  args.append(GraphNodeExpression.evaluate(arg, context))
152  argstrings.append(arg)
153  #print >> sys.stderr, 'RETURNING [%s] [%s]' % (args, argstrings)
154  return (args, argstrings)
155 
156  @staticmethod
157  def functioncall(expression, context):
158  '''Performs a function call for our expression language
159 
160  Figures out the function name, and the arguments and then
161  calls that function with those arguments.
162 
163  All our defined functions take an argv argument string first, then an
164  ExpressionContext argument.
165 
166  This parsing is incredibly primitive. Feel free to improve it ;-)
167 
168  '''
169  expression = expression.strip()
170  if expression[-1] != ')':
171  print >> sys.stderr, '%s does not end in )' % expression
172  return None
173  expression = expression[:len(expression)-1]
174  (funname, arglist) = expression.split('(', 1)
175  # print >> sys.stderr, 'FUNCTIONCALL: %s(%s)' % (funname, arglist)
176  funname = funname.strip()
177  arglist = arglist.strip()
178  #
179  # At this point we have all our arguments as a string , but it might contain
180  # other (nested) calls for us to evaluate
181  #
182  # print >> sys.stderr, 'FunctionCall: arglist: [%s]' % (arglist)
183  args, _argstrings = GraphNodeExpression._compute_function_args(arglist, context)
184  # print >> sys.stderr, 'args: %s' % (args)
185  # print >> sys.stderr, '_argstrings: %s' % (_argstrings)
186  if args is None:
187  return None
188 
189  if funname.startswith('@'):
190  funname = funname[1:]
191  if funname not in GraphNodeExpression.functions:
192  print >> sys.stderr, 'BAD FUNCTION NAME: %s' % funname
193  return None
194  # print >> sys.stderr, 'ARGSTRINGS %s(%s)' % (funname, str(_argstrings))
195  # print >> sys.stderr, 'ARGS: %s' % (str(args))
196  ret = GraphNodeExpression.functions[funname](args, context)
197  # print >> sys.stderr, '%s(%s) => %s' % (funname, args, ret)
198  return ret
199 
200  @staticmethod
202  '''Return a list of tuples of (funcname, docstring) for all our GraphNodeExpression
203  defined functions. The list is sorted by function name.
204  '''
205  names = GraphNodeExpression.functions.keys()
206  names.sort()
207  ret = []
208  for name in names:
209  ret.append((name, inspect.getdoc(GraphNodeExpression.functions[name])))
210  return ret
211 
212  @staticmethod
213  def RegisterFun(function):
214  'Function to register other functions as built-in GraphNodeExpression functions'
215  GraphNodeExpression.functions[function.__name__] = function
216  return function
217 
218 
219 class ExpressionContext(object):
220  '''This class defines a context for an expression evaluation.
221  There are three parts to it:
222  1) A cache of values which have already been computed
223  2) A scope/context for expression evaluation - a default name prefix
224  3) A set of objects which implement the 'get' operation to be used in
225  evaluating values of names
226 
227  We act like a dict, implementing these member functions:
228  __iter__, __contains__, __len__, __getitem__ __setitem__, __delitem__,
229  get, keys, has_key, clear, items
230  '''
231 
232  def __init__(self, objects, prefix=None):
233  'Initialize our ExpressionContext'
234  self.objects = objects if isinstance(objects, (list, tuple)) else (objects,)
235  self.prefix = prefix
236  self.values = {}
237 
238  def __str__(self):
239  ret = 'ExpressionContext('
240  delim='['
241  for obj in self.objects:
242  ret += ('%s%s' % (delim, str(obj)))
243  delim=', '
244  ret += '])'
245  return ret
246 
247 
248  def keys(self):
249  '''Return the complete set of keys in all our constituent objects'''
250  retkeys = set()
251  for obj in self.objects:
252  for key in obj:
253  retkeys.add(key)
254  return retkeys
255 
256  @staticmethod
257  def _fixvalue(v):
258  'Fix up a return value to avoid unicode values...'
259  if isinstance(v, unicode):
260  return str(v)
261  if not isinstance(v, str) and hasattr(v, '__iter__') and not hasattr(v, '__getitem__'):
262  ret = []
263  for item in v:
264  ret.append(ExpressionContext._fixvalue(item))
265  return ret
266  return v
267 
268  def get(self, key, alternative=None):
269  '''Return the value associated with a key - cached or otherwise
270  and cache it.'''
271  if key in self.values:
272  return self.values[key]
273  for obj in self.objects:
274  ret = None
275  try:
276  #print >> sys.stderr, 'GETTING %s in %s: %s' % (key, type(obj), obj)
277  ret = obj.get(key, None)
278  if ret is None and hasattr(obj, 'deepget'):
279  ret = obj.deepget(key, None)
280  if isinstance(ret, unicode):
281  ret = str(ret)
282  #print >> sys.stderr, 'RETURNED %s' % ret
283  # Too general exception catching...
284  # pylint: disable=W0703
285  except Exception as e:
286  ret = None
287  print >> sys.stderr, 'OOPS: self.objects = %s / exception %s' % (str(self.objects),
288  e)
289  print >> sys.stderr, 'OOPS: OUR object = %s (%s)' % (str(obj), type(obj))
290  ret = ExpressionContext._fixvalue(ret)
291  if ret is not None:
292  self.values[key] = ret
293  return ret
294  if self.prefix is not None:
295  ret = ExpressionContext._fixvalue(obj.get('%s.%s' % (self.prefix, key), None))
296  if ret is not None:
297  self.values[key] = ret
298  return ret
299  return alternative
300 
301  def clear(self):
302  'Clear our cached values'
303  self.values = {}
304 
305  def items(self):
306  'Return all items from our cache'
307  return self.values.items()
308 
309 
310  def __iter__(self):
311  'Yield each key from self.keys() in turn'
312  for key in self.keys():
313  yield key
314 
315  def __contains__(self, key):
316  'Return True if we can get() this key'
317  return self.get(key, None) is not None
318 
319  def has_key(self, key):
320  'Return True if we can get() this key'
321  return self.get(key, None) is not None
322 
323 
324  def __len__(self):
325  'Return the number of keys in our objects'
326  return len(self.keys())
327 
328  def __getitem__(self, key):
329  'Return the given item, or raise KeyError if not found'
330  ret = self.get(key, None)
331  if ret is None:
332  raise KeyError(key)
333  return ret
334 
335  def __setitem__(self, key, value):
336  'Cache the value associated with this key'
337  self.values[key] = value
338 
339  def __delitem__(self, key):
340  'Remove the cache value associated with this key'
341  del self.values[key]
342 
343 @GraphNodeExpression.RegisterFun
344 def IGNORE(_ignoreargs, _ignorecontext):
345  '''Function to ignore its argument(s) and return True all the time.
346  This is a special kind of no-op in that it is used to override
347  and ignore an underlying rule. It is expected that its arguments
348  will explain why it is being ignored in this rule set.
349  '''
350  return True
351 
352 @GraphNodeExpression.RegisterFun
353 def EQ(args, _context):
354  '''Function to return True if each non-None argument in the list matches
355  every non-None argument and at least one of its subsequent arguments are not None.
356  '''
357  #print >> sys.stderr, 'EQ(%s) =>?' % str(args)
358  val0 = args[0]
359  if val0 is None:
360  return None
361  anymatch = None
362  for val in args[1:]:
363  if val is None:
364  continue
365  if not isinstance(val, type(val0)):
366  if str(val0) != str(val):
367  return False
368  elif val0 != val:
369  return False
370  anymatch = True
371  #print >> sys.stderr, 'EQ(%s) => %s' % (str(args), str(anymatch))
372  return anymatch
373 
374 @GraphNodeExpression.RegisterFun
375 def NE(args, _context):
376  '''Function to return True if no non-None argument in the list matches
377  the first one or None if all subsequent arguments are None'''
378  #print >> sys.stderr, 'NE(%s, %s)' % (args[0], str(args[1:]))
379  val0 = args[0]
380  if val0 is None:
381  return None
382  anymatch = None
383  for val in args[1:]:
384  #print >> sys.stderr, '+NE(%s, %s) (%s, %s)' % (val0, val, type(val0), type(val))
385  if val is None:
386  return None
387  if val0 == val or str(val0) == str(val):
388  return False
389  anymatch = True
390  return anymatch
391 
392 @GraphNodeExpression.RegisterFun
393 def LT(args, _context):
394  '''Function to return True if each non-None argument in the list is
395  less than the first one or None if all subsequent arguments are None'''
396  val0 = args[0]
397  if val0 is None:
398  return None
399  anymatch = None
400  for val in args[1:]:
401  if val is None:
402  continue
403  if val0 >= val:
404  return False
405  anymatch = True
406  return anymatch
407 
408 @GraphNodeExpression.RegisterFun
409 def GT(args, _context):
410  '''Function to return True if each non-None argument in the list is
411  greater than the first one or None if all subsequent arguments are None'''
412  val0 = args[0]
413  if val0 is None:
414  return None
415  anymatch = None
416  for val in args[1:]:
417  if val is None:
418  continue
419  if val0 <=val:
420  return False
421  anymatch = True
422  return anymatch
423 
424 @GraphNodeExpression.RegisterFun
425 def LE(args, _context):
426  '''Function to return True if each non-None argument in the list is
427  less than or equal to first one or None if all subsequent arguments are None'''
428  val0 = args[0]
429  if val0 is None:
430  return None
431  anymatch = None
432  for val in args[1:]:
433  if val is None:
434  continue
435  if val0 > val:
436  return False
437  anymatch = True
438  return anymatch
439 
440 @GraphNodeExpression.RegisterFun
441 def GE(args, _context):
442  '''Function to return True if each non-None argument in the list is
443  greater than or equal to first one or None if all subsequent arguments are None'''
444  val0 = args[0]
445  if val0 is None:
446  return None
447  anymatch = None
448  for val in args[1:]:
449  if val is None:
450  continue
451  if val0 < val:
452  return False
453  anymatch = True
454  return anymatch
455 
456 @GraphNodeExpression.RegisterFun
457 def IN(args, _context):
458  '''Function to return True if first argument is in the list that follows.
459  If the first argument is iterable, then each element in it must be 'in'
460  the list that follows.
461  '''
462 
463  val0 = args[0]
464  if val0 is None:
465  return None
466  if hasattr(val0, '__iter__') and not isinstance(val0, (str, unicode)):
467  # Iterable
468  anyTrue = False
469  for elem in val0:
470  if elem is None:
471  continue
472  if elem not in args[1:] and str(elem) not in args[1:]:
473  return False
474  anyTrue = True
475  return True if anyTrue else None
476  # Not an iterable: string, int, NoneType, etc.
477  if val0 is None:
478  return None
479  #print >> sys.stderr, type(val0), val0, type(args[1]), args[1]
480  return val0 in args[1:] or str(val0) in args[1:]
481 
482 @GraphNodeExpression.RegisterFun
483 def NOTIN(args, _context):
484  'Function to return True if first argument is NOT in the list that follows'
485  val0 = args[0]
486  if val0 is None:
487  return None
488  if hasattr(val0, '__iter__') and not isinstance(val0, (str, unicode)):
489  # Iterable
490  for elem in val0:
491  if elem in args[1:] or str(elem) in args[1:]:
492  return False
493  return True
494  return val0 not in args[1:] and str(val0) not in args[1:]
495 
496 @GraphNodeExpression.RegisterFun
497 def NOT(args, _context):
498  'Function to Negate the Truth value of its single argument'
499  try:
500  val0 = args[0]
501  except TypeError:
502  val0 = args
503  return None if val0 is None else not val0
504 
505 
506 def _str_to_regexflags(s):
507  r'''Transform a string of single character regex flags to the corresponding integer.
508  Note that the flag names are all the Python single character flag names from the 're' module.
509  They are as follows:
510  A perform 8-bit ASCII-only matching (Python 3 only)
511  I Perform non-case-sensitive matching
512  L Use locale settings for \w, =W, \b and \B
513  M Multi-line match - allow ^ and $ to apply to individual lines in the string
514  S Allow the dot character to also match a newline
515  U Uses information from the Unicode character properties for \w, \W, \b and \B.
516  (python 2 only)
517  X Ignores unescaped whitespace and comments in the pattern string.
518  '''
519 
520  flags = 0
521  if s is not None:
522  for char in s:
523  if char == 'A':
524  if hasattr(re, 'ASCII'):
525  flags |= getattr(re, 'ASCII')
526  elif char == 'I':
527  flags |= re.IGNORECASE
528  elif char == 'L':
529  flags |= re.LOCALE
530  elif char == 'M':
531  flags |= re.MULTILINE
532  elif char == 'S':
533  flags |= re.DOTALL
534  elif char == 'U':
535  flags |= re.UNICODE
536  elif char == 'X':
537  flags |= re.VERBOSE
538  return flags
539 
540 _regex_cache = {}
541 def _compile_and_cache_regex(regexstr, flags=None):
542  'Compile and cache a regular expression with the given flags'
543  cache_key = '%s//%s' % (str(regexstr), str(flags))
544  if cache_key in _regex_cache:
545  regex = _regex_cache[cache_key]
546  else:
547  regex = re.compile(regexstr, _str_to_regexflags(flags))
548  _regex_cache[cache_key] = regex
549  return regex
550 
551 @GraphNodeExpression.RegisterFun
552 def match(args, _context):
553  '''Function to return True if first argument matches the second argument (a regex)
554  - optional 3rd argument is RE flags'''
555  lhs = str(args[0])
556  rhs = args[1]
557  if lhs is None or rhs is None:
558  return None
559  flags = args[2] if len(args) > 2 else None
560  regex = _compile_and_cache_regex(rhs, flags)
561  return regex.search(lhs) is not None
562 
563 @GraphNodeExpression.RegisterFun
564 def argequals(args, context):
565  '''
566  usage: argequals name-to-search-for [list-to-search]
567 
568  A function which searches a list for an argument of the form name=value.
569  The value '$argv' is the default name of the list to search.
570  If there is a second argument, then that second argument is an expression
571  expected to yield an iterable to search in for the name=value string instead of '$argv'
572  '''
573  #print >> sys.stderr, 'ARGEQUALS(%s)' % (str(args))
574  if len(args) > 2 or len(args) < 1:
575  return None
576  definename = args[0]
577  argname = args[1] if len(args) >= 2 else '$argv'
578  listtosearch = GraphNodeExpression.evaluate(argname, context)
579  #print >> sys.stderr, 'SEARCHING in %s FOR %s in %s' % (argname, definename, listtosearch)
580  if listtosearch is None:
581  return None
582  prefix = '%s=' % definename
583  # W0702: No exception type specified for except statement
584  # pylint: disable=W0702
585  try:
586  for elem in listtosearch:
587  if elem.startswith(prefix):
588  return elem[len(prefix):]
589  except: # No matter the cause of failure, return None...
590  pass
591  return None
592 
593 @GraphNodeExpression.RegisterFun
594 def argmatch(args, context):
595  '''
596  usage: argmatch regular-expression [list-to-search [regex-flags]]
597 
598  Argmatch searches a list for an value that matches a given regex.
599  The regular expression is given by the argument in args, and the list 'argv'
600  defaults to be the list of arguments to be searched.
601 
602  If there are two arguments in args, then the first argument is the
603  array value to search in for the regular expression string instead of 'argv'
604 
605  If the regex contains a parenthesized groups, then the value of the first such group
606  is returned, otherwise the part of the argument that matches the regex is returned.
607 
608  Note that this regular expression is 'anchored' that is, it starts with the first character
609  in the argument. If you want it to be floating, then you may want to start your regex
610  with '.*' and possibly parenthesize the part you want to return.
611  '''
612  #print >> sys.stderr, 'ARGMATCH(%s)' % (str(args))
613  #print >> sys.stderr, 'ARGMATCHCONTEXT(%s)' % (str(context))
614  if len(args) > 3 or len(args) < 1:
615  return None
616  regexstr = args[0]
617  argname = args[1] if len(args) >= 2 else '$argv'
618  flags = args[2] if len(args) >= 3 else None
619  listtosearch = GraphNodeExpression.evaluate(argname, context)
620  if listtosearch is None:
621  return None
622 
623  # W0702: No exception type specified for except statement
624  # pylint: disable=W0702
625  try:
626  #print >>sys.stderr, 'Compiling regex: /%s/' % regexstr
627  regex = _compile_and_cache_regex(regexstr, flags)
628  #print >>sys.stderr, 'Matching against list %s' % (str(listtosearch))
629  for elem in listtosearch:
630  #print >>sys.stderr, 'Matching %s against %s' % (regexstr, elem)
631  matchobj = regex.match(elem)
632  if matchobj:
633  # Did they specify any parenthesized groups?
634  if len(matchobj.groups()) > 0:
635  # yes - return the (first) parenthesized match
636  return matchobj.groups()[0]
637  else:
638  # no - return everything matched
639  return matchobj.group()
640  except: # No matter the cause of failure, return None...
641  # That includes ill-formed regular expressions...
642  pass
643  return None
644 
645 @GraphNodeExpression.RegisterFun
646 def flagvalue(args, context):
647  '''
648  usage: flagvalue flag-name [list-to-search]
649  A function which searches a list for a -flag and returns
650  the value of the string which is the next argument.
651  The -flag is given by the argument in args, and the list 'argv'
652  is assumed to be the list of arguments.
653  If there are two arguments in args, then the first argument is the
654  array value to search in for the -flag string instead of 'argv'
655  The flag given must be the entire flag complete with - character.
656  For example -X or --someflag.
657  '''
658  if len(args) > 2 or len(args) < 1:
659  return None
660  flagname = args[0]
661  argname = args[1] if len(args) >= 2 else '$argv'
662 
663  progargs = GraphNodeExpression.evaluate(argname, context)
664  argslen = len(progargs)
665  flaglen = len(flagname)
666  for pos in range(0, argslen):
667  progarg = progargs[pos]
668  progarglen = len(progarg)
669  if progarg.startswith(flagname):
670  if progarg == flagname:
671  # -X foobar
672  if (pos+1) < argslen:
673  return progargs[pos+1]
674  elif flaglen == 2 and progarglen > flaglen:
675  # -Xfoobar -- single character flags only
676  return progarg[2:]
677  return None
678 
679 @GraphNodeExpression.RegisterFun
680 def OR(args, context):
681  '''
682  A function which evaluates each expression in turn, and returns the value
683  of the first expression which is not None - or None
684  '''
685  #print >> sys.stderr, 'OR(%s)' % (str(args))
686  if len(args) < 1:
687  return None
688  anyfalse = False
689  for arg in args:
690  value = GraphNodeExpression.evaluate(arg, context)
691  if value is not None:
692  if value:
693  return value
694  else:
695  anyfalse = True
696  return False if anyfalse else None
697 
698 @GraphNodeExpression.RegisterFun
699 def AND(args, context):
700  '''
701  A function which evaluates each expression in turn, and returns the value
702  of the first expression which is not None - or None
703  '''
704  # print >> sys.stderr, 'AND(%s)' % (str(args))
705  argisnone = True
706  if len(args) < 1:
707  return None
708  for arg in args:
709  value = GraphNodeExpression.evaluate(arg, context)
710  if value is None:
711  argisnone = None
712  elif not value:
713  # print >> sys.stderr, 'AND(%s) => False' % (str(args))
714  return False
715  # print >> sys.stderr, 'AND(%s) => %s' % (str(args), argisnone)
716  return argisnone
717 
718 @GraphNodeExpression.RegisterFun
719 def ATTRSEARCH(args, context):
720  '''
721  Search our first context object for an attribute with the given name and (if supplied) value.
722  If 'value' is None, then we simply search for the given name.
723  We return True if we found what we were looking for, and False otherwise.
724 
725  The object to search in is is args[0], the name is args[1],
726  and the optional desired value is args[2].
727  '''
728  return True if FINDATTRVALUE(args, context) else False
729  # return FINDATTRVALUE(args, context) is not None
730  # These are equivalent. Not sure which is clearer...
731 
732 @GraphNodeExpression.RegisterFun
733 def FINDATTRVALUE(args, _context):
734  '''
735  Search our first context object for an attribute with the given name and (if supplied) value.
736  We return the value found, if it is in the context objects, or None if it is not
737  If 'value' is None, then we simply search for the given name.
738 
739  We return True if the desired value is None, and so is the value we found -
740  otherwise we return the value associated with 'name' or None if not found.
741 
742  The object to search in is is args[0], the name is args[1],
743  and the optional desired value is args[2].
744  '''
745  if len(args) not in (2,3):
746  print >> sys.stderr, 'WRONG NUMBER OF ARGUMENTS (%d) TO FINDATTRVALUE' % (len(args))
747  return None
748  desiredvalue = args[2] if len(args) > 2 else None
749  return _attrfind(args[0], args[1], desiredvalue)
750 
751 def _is_scalar(obj):
752  'Return True if this object is a pyConfigContext/JSON "scalar"'
753  return isinstance(obj, (str, unicode, int, long, float, bool, pyNetAddr))
754 
755 def _attrfind(obj, name, desiredvalue):
756  '''
757  Recursively search the given object for an attribute with the given name
758  and value. If 'value' is None, then we simply search for the given name.
759 
760  We return True if the desired value is None, and the value we found is also None -
761  otherwise we return the value associated with 'name' or None if not found.
762  '''
763  if _is_scalar(obj):
764  return None
765  if hasattr(obj, '__getitem__'):
766  for key in obj:
767  keyval = obj[key]
768  if key == name:
769  if desiredvalue is None:
770  return keyval if keyval is not None else True
771  elif keyval == desiredvalue or str(keyval) == str(desiredvalue):
772  # We use str() to allow pyNetAddr objects to compare equal
773  # and the possibility of type mismatches (strings versus integers, for example)
774  # This may also improve the chance of floating point compares working as
775  # intended.
776  return keyval
777  elif hasattr(obj, '__iter__'):
778  for elem in obj:
779  ret = _attrfind(elem, name, desiredvalue)
780  if ret is not None:
781  return ret
782  return None
783 
784 @GraphNodeExpression.RegisterFun
785 def PAMMODARGS(args, _context):
786  '''
787  We pass the following arguments to PAMSELECTARGS:
788  section - the section value to select from
789  service - service type to search for
790  module - the module to select arguments from
791  argument - the arguments to select
792 
793  We return the arguments from the first occurence of the module that we find.
794  '''
795  #print >> sys.stderr, 'PAMMODARGS(%s)' % (str(args))
796  if len(args) != 4:
797  print >> sys.stderr, 'WRONG NUMBER OF ARGUMENTS (%d) TO PAMMODARGS' % (len(args))
798  return False
799  section = args[0]
800  reqservice = args[1]
801  reqmodule = args[2]
802  reqarg = args[3]
803 
804  if section is None:
805  #print >> sys.stderr, 'Section is None in PAM object'
806  return None
807  # Each section is a list of lines
808  for line in section:
809  # Each line is a dict with potential keys of:
810  # - service: a keyword saying what kind of service
811  # - filename:(only for includes)
812  # - type: dict of keywords (requisite, required, optional, etc)
813  # - module: Module dict keywords with:
814  # - path - pathname of module ending in .so
815  # - other arguments as per the module's requirements
816  # simple flags without '=' values show up with True as value
817  #
818  if 'service' not in line or line['service'] != reqservice:
819  #print >> sys.stderr, 'Service %s not in PAM line %s' % (reqservice, str(line))
820  continue
821  if 'module' not in line:
822  #print >> sys.stderr, '"module" not in PAM line %s' % str(line)
823  continue
824  if 'path' not in line['module']:
825  #print >> sys.stderr, '"path" not in PAM module %s' % str(line['module'])
826  #print >> sys.stderr, '"path" not in PAM line %s' % str(line)
827  continue
828  modargs = line['module']
829  if reqmodule != 'ANY' and (modargs['path'] != reqmodule and
830  modargs['path'] != (reqmodule + '.so')):
831  #print >> sys.stderr, 'Module %s not in PAM line %s' % (reqmodule, str(line))
832  continue
833  ret = modargs[reqarg] if reqarg in modargs else None
834  if ret is None and reqmodule == 'ANY':
835  continue
836  #print >> sys.stderr, 'RETURNING %s from %s' % (ret, str(line))
837  return ret
838  return None
839 
840 
841 
842 @GraphNodeExpression.RegisterFun
843 def MUST(args, _unused_context):
844  'Return True if all args are True. A None arg is the same as False to us'
845  # print >> sys.stderr, 'CALLING MUST%s' % str(tuple(args))
846  if not hasattr(args, '__iter__') or isinstance(args, (str, unicode)):
847  args = (args,)
848  for arg in args:
849  if arg is None or not arg:
850  #print >> sys.stderr, '+++MUST returns FALSE'
851  return False
852  # print >> sys.stderr, '+++MUST returns TRUE'
853  return True
854 
855 @GraphNodeExpression.RegisterFun
856 def NONEOK(args, _unused_context):
857  'Return True if all args are True or None - that is, if no args are False'
858  #print >> sys.stderr, 'CALLING MUST%s' % str(tuple(args))
859  if not hasattr(args, '__iter__') or isinstance(args, (str, unicode)):
860  args = (args,)
861  for arg in args:
862  if arg is not None and not arg:
863  #print >> sys.stderr, '+++NONEOK returns FALSE'
864  return False
865  #print >> sys.stderr, '+++NONEOK returns TRUE'
866  return True
867 @GraphNodeExpression.RegisterFun
868 def FOREACH(args, context):
869  '''Applies the (string) expression (across all values in the context,
870  returning the 'AND' of the evaluation of the expression-evaluations
871  across the top level values in the context. It stops evaluation on
872  the first False return.
873 
874  The final argument is the expression (predicate) to be evaluated. Any
875  previous arguments in 'args' are expressions to be evaluated in the context
876  'context' then used as the 'context' for this the expression in this FOREACH.
877  Note that this desired predicate is a _string_, which is then evaluated
878  (like 'eval'). It is not a normal expression, but a string containing
879  an expression. You _will_ have to quote it.
880 
881  When given a single argument, it will evaluate the string expression
882  for each of top-level values in the object. Normally this would be the 'data'
883  portion of a discovery object. So, for example, if each of the top level keys
884  is a file name and the values are file properties, then it will evaluate the
885  expression on the properties of every file in the object.
886 
887  If you need to evaluate this across all the elements of a sub-object named
888  "filenames" in the top level "data" object then you give "$filenames" as the
889  context argument, and your predicate as the expression like this:
890  ["$filenames", "<your-desired-predicate>"].
891 
892  The code to do this is simpler than the explanation ;-)
893  '''
894  anynone = False
895  if len(args) == 1:
896  objectlist = context.objects
897  else:
898  objectlist = [GraphNodeExpression.evaluate(obj, context) for obj in args[:-1]]
899 
900  expressionstring=args[-1]
901  if not isinstance(expressionstring, (str, unicode)):
902  print >> sys.stderr, 'FOREACH expression must be a string, not %s' % type(expressionstring)
903  return False
904  # print >>sys.stderr, 'OBJECTLIST is:', objectlist
905  for obj in objectlist:
906  # print >>sys.stderr, 'OBJ is:', obj
907  for key in obj:
908  item = obj[key]
909  if not hasattr(item, '__contains__') or not hasattr(item, '__iter__'):
910  print >> sys.stderr, 'UNSUITABLE FOREACH CONTEXT[%s]: %s' % (key, item)
911  continue
912  # print >> sys.stderr, 'CREATING CONTEXT[%s]: %s' % (key, item)
913  itemcontext = ExpressionContext(item)
914  # print >> sys.stderr, 'CONTEXT IS:', itemcontext
915  value = GraphNodeExpression.evaluate(expressionstring, itemcontext)
916  # print >> sys.stderr, 'VALUE of %s IS [%s] in context: %s' % (str(args), value, item)
917  if value is None:
918  anynone = True
919  elif not value:
920  return False
921  return None if anynone else True
922 
923 
924 
925 @GraphNodeExpression.RegisterFun
926 def bitwiseOR(args, context):
927  '''
928  A function which evaluates the each expression and returns the bitwise OR of
929  all the expressions given as arguments
930  '''
931  if len(args) < 2:
932  return None
933  result = 0
934  for arg in args:
935  value = GraphNodeExpression.evaluate(arg, context)
936  if value is None:
937  return None
938  result |= int(value)
939  return result
940 
941 @GraphNodeExpression.RegisterFun
942 def bitwiseAND(args, context):
943  '''
944  A function which evaluates the each expression and returns the bitwise AND of
945  all the expressions given as arguments
946  '''
947  if len(args) < 2:
948  return None
949  result = int(args[0])
950  for arg in args:
951  value = GraphNodeExpression.evaluate(arg, context)
952  if value is None:
953  return None
954  result &= int(value)
955  return result
956 
957 @GraphNodeExpression.RegisterFun
958 def is_upstartjob(args, context):
959  '''
960  Returns "true" if any of its arguments names an upstart job, "false" otherwise
961  If no arguments are given, it returns whether this system has upstart enabled.
962  '''
963 
964 
965  from monitoring import MonitoringRule
966  agentcache = MonitoringRule.compute_available_agents(context)
967 
968  if 'upstart' not in agentcache or len(agentcache['upstart']) == 0:
969  return 'false'
970 
971  for arg in args:
972  value = GraphNodeExpression.evaluate(arg, context)
973  if value in agentcache['upstart']:
974  return 'true'
975  return len(args) == 0
976 
977 def _regexmatch(key):
978  '''Handy internal function to pull out the IP and port into a pyNetAddr
979  Note that the format is the format used in the discovery information
980  which in turn is the format used by netstat.
981  This is not a "standard" format, but it's what netstat uses - so it's
982  what we use.
983  '''
984  mobj = ipportregex.match(key)
985  if mobj is None:
986  return None
987  (ip, port) = mobj.groups()
988  ipport = pyNetAddr(ip, port=int(port))
989  if ipport.isanyaddr():
990  if ipport.addrtype() == ADDR_FAMILY_IPV4:
991  ipport = pyNetAddr('127.0.0.1', port=ipport.port())
992  else:
993  ipport = pyNetAddr('::1', port=ipport.port())
994  return ipport
995 
996 def _collect_ip_ports(service):
997  'Collect out complete set of IP/Port combinations for this service'
998  portlist = {}
999  for key in service.keys():
1000  ipport = _regexmatch(key)
1001  if ipport.port() == 0:
1002  continue
1003  port = ipport.port()
1004  if port in portlist:
1005  portlist[port].append(ipport)
1006  else:
1007  portlist[port] = [ipport,]
1008  return portlist
1009 
1010 # Netstat format IP:port pattern
1011 ipportregex = re.compile('(.*):([^:]*)$')
1012 def selectanipport(arg, _context, preferlowestport=True, preferv4=True):
1013  '''This function searches discovery information for a suitable IP
1014  address/port combination to go with the service.
1015  '''
1016 
1017  #print >> sys.stderr, 'SELECTANIPPORT(%s)' % arg
1018  try:
1019 
1020  portlist = _collect_ip_ports(arg)
1021  portkeys = portlist.keys()
1022  if preferlowestport:
1023  portkeys.sort()
1024  for p in portlist[portkeys[0]]:
1025  if preferv4:
1026  if p.addrtype() == ADDR_FAMILY_IPV4:
1027  return p
1028  else:
1029  if p.addrtype() == ADDR_FAMILY_IPV6:
1030  return p
1031  return portlist[portkeys[0]][0]
1032  except (KeyError, ValueError, TypeError, IndexError):
1033  # Something is hinky with this data
1034  return None
1035 
1036 @GraphNodeExpression.RegisterFun
1037 def serviceip(args, context):
1038  '''
1039  This function searches discovery information for a suitable concrete IP
1040  address for a service.
1041  The argument to this function tells it an expression that will give
1042  it the hash table (map) of IP/port combinations for this service.
1043  '''
1044  if len(args) == 0:
1045  args = ('$procinfo.listenaddrs',)
1046  #print >> sys.stderr, 'SERVICEIP(%s)' % str(args)
1047  for arg in args:
1048  nmap = GraphNodeExpression.evaluate(arg, context)
1049  if nmap is None:
1050  continue
1051  #print >> sys.stderr, 'serviceip.SELECTANIPPORT(%s)' % (nmap)
1052  ipport = selectanipport(nmap, context)
1053  if ipport is None:
1054  continue
1055  ipport.setport(0) # Make sure return value doesn't include the port
1056  #print >> sys.stderr, 'IPPORT(%s)' % str(ipport)
1057  return str(ipport)
1058  return None
1059 
1060 @GraphNodeExpression.RegisterFun
1061 def serviceport(args, context):
1062  '''
1063  This function searches discovery information for a suitable port for a service.
1064  The argument to this function tells it an expression that will give
1065  it the hash table (map) of IP/port combinations for this service.
1066  '''
1067  if len(args) == 0:
1068  args = ('$procinfo.listenaddrs',)
1069  #print >> sys.stderr, 'SERVICEPORT ARGS are %s' % (str(args))
1070  for arg in args:
1071  nmap = GraphNodeExpression.evaluate(arg, context)
1072  if nmap is None:
1073  continue
1074  port = selectanipport(nmap, context).port()
1075  if port is None:
1076  continue
1077  return str(port)
1078  return None
1079 
1080 @GraphNodeExpression.RegisterFun
1081 def serviceipport(args, context):
1082  '''
1083  This function searches discovery information for a suitable ip:port combination.
1084  The argument to this function tells it an expression that will give
1085  it the hash table (map) of IP/port combinations for this service.
1086  The return value is a legal ip:port combination for the given
1087  address type (ipv4 or ipv6)
1088  '''
1089  if len(args) == 0:
1090  args = ('$procinfo.listenaddrs',)
1091  for arg in args:
1092  nmap = GraphNodeExpression.evaluate(arg, context)
1093  if nmap is None:
1094  continue
1095  ipport = selectanipport(nmap, context)
1096  if ipport is None:
1097  continue
1098  return str(ipport)
1099  return None
1100 
1101 @GraphNodeExpression.RegisterFun
1102 def basename(args, context):
1103  '''
1104  This function returns the basename from a pathname.
1105  If no pathname is supplied, then the executable name is assumed.
1106  '''
1107  if isinstance(args, (str, unicode)):
1108  args = (args,)
1109  if len(args) == 0:
1110  args = ('$pathname',) # Default to the name of the executable
1111  for arg in args:
1112  pathname = GraphNodeExpression.evaluate(arg, context)
1113  if pathname is None:
1114  continue
1115  #print >> sys.stderr, 'BASENAME(%s) => %s' % ( pathname
1116  #, os.path.basename(pathname))
1117  return os.path.basename(pathname)
1118  return None
1119 
1120 
1121 @GraphNodeExpression.RegisterFun
1122 def dirname(args, context):
1123  '''
1124  This function returns the directory name from a pathname.
1125  If no pathname is supplied, then the discovered service executable name is assumed.
1126  '''
1127  if isinstance(args, (str, unicode)):
1128  args = (args,)
1129  if len(args) == 0:
1130  args = ('$pathname',) # Default to the name of the executable
1131  for arg in args:
1132  pathname = GraphNodeExpression.evaluate(arg, context)
1133  if pathname is None:
1134  continue
1135  return os.path.dirname(pathname)
1136  return None
1137 
1138 @GraphNodeExpression.RegisterFun
1139 def hascmd(args, context):
1140  '''
1141  This function returns True if the given list of commands are all present on the given Drone.
1142  It determines this by looking at the value of $_init_commands.data
1143  '''
1144  cmdlist = GraphNodeExpression.evaluate('$_init_commands.data', context)
1145  for arg in args:
1146  if cmdlist is None or arg not in cmdlist:
1147  return None
1148  return True
1149 
1150 if __name__ == '__main__':
1151 
1153  '''These tests don't require a real context'''
1154  assert NOT((True,), None) is False
1155  assert NOT((False,), None) is True
1156  assert EQ((1,1,'1'), None) is True
1157  assert NOT(EQ((1,), None), None) is None
1158  assert MUST(NOT(EQ((1,), None), None), None) is False
1159  assert NONEOK(NOT(EQ((1,), None), None), None) is True
1160  assert NOT(EQ((1,1,'2'), None), None) is True
1161  assert NOT(EQ((0,0,'2'), None), None) is True
1162  assert EQ(('a','a','a'), None) is True
1163  assert EQ(('0','0',0), None) is True
1164  assert NOT(NE((1,1,'1'), None), None) is True
1165  assert NOT(NE((1,), None), None) is None
1166  assert NONEOK(NOT(NE((1,), None), None), None) is True
1167  assert MUST(NOT(NE((1,), None), None), None) is False
1168  assert NOT(NE((1,1,'2'), None), None) is True
1169  assert NOT(NE((0,0,'2'), None), None) is True
1170  assert NOT(NE(('a','a','a'), None), None) is True
1171  assert NOT(NE(('0','0',0), None), None) is True
1172  assert LE((1,1), None) is True
1173  assert LE((1,5), None) is True
1174  assert NOT(LT((1,1), None), None) is True
1175  assert LT((1,5), None) is True
1176  assert NOT(GT((1,1), None), None) is True
1177  assert GE((1,1), None) is True
1178  assert IN ((1, 2 , 3, 4, 1), None) is True
1179  assert IN ((1, 2 , 3, 4, '1'), None) is True
1180  assert NOT(IN((1, 2 , 3, 4), None), None) is True
1181  assert NOT(NOTIN((1, 2 , 3, 4, 1), None), None) is True
1182  assert NOT(NOTIN((1, 2 , 3, 4, '1'), None), None) is True
1183  assert NOTIN((1, 2 , 3, 4), None) is True
1184  assert bitwiseOR((1, 2, 4), None) == 7
1185  assert bitwiseOR((1, 2, '4'), None) == 7
1186  assert bitwiseAND((7, 3), None) == 3
1187  assert bitwiseAND((7, 1, '2'), None) == 0
1188  assert bitwiseAND(('15', '7', '3'), None) == 3
1189  assert IGNORE((False, False, False), None)
1190  assert MUST(None, None) is False
1191  assert MUST(True, None) is True
1192  assert MUST(False, None) is False
1193  assert NONEOK(None, None) is True
1194  assert NONEOK(True, None) is True
1195  assert NONEOK(False, None) is False
1196  assert match(('fred', 'fre'), None)
1197  assert match(('fred', 'FRE'), None) is False
1198  assert match(('fred', 'FRE', 'I'), None) is True
1199  assert basename(('/dev/null'), None) == 'null'
1200  assert dirname(('/dev/null'), None) == '/dev'
1201  print >> sys.stderr, 'Simple tests passed.'
1202 
1204  'GraphNodeExpression tests that need a context'
1205 
1206  lsattrs='''{
1207  "/var/log/audit/": {"owner": "root", "group": "root", "type": "d", "perms": {"owner":{"read":true, "write":true, "exec":true, "setid":false}, "group": {"read":true, "write":false, "exec":true, "setid":false}, "other": {"read":false, "write":false, "exec":false}, "sticky":false}, "octal": "0750"},
1208  "/var/log/audit/audit.log": {"owner": "root", "group": "root", "type": "-", "perms": {"owner":{"read":true, "write":true, "exec":false, "setid":false}, "group": {"read":false, "write":false, "exec":false, "setid":false}, "other": {"read":false, "write":false, "exec":false}, "sticky":false}, "octal": "0600"},
1209  "/var/log/audit/audit.log.1": {"owner": "root", "group": "root", "type": "-", "perms": {"owner":{"read":true, "write":false, "exec":false, "setid":false}, "group": {"read":false, "write":false, "exec":false, "setid":false}, "other": {"read":false, "write":false, "exec":false}, "sticky":false}, "octal": "0400"}
1210 }'''
1211  lscontext = ExpressionContext(pyConfigContext(lsattrs,))
1212 
1213  Pie_context = ExpressionContext((
1214  pyConfigContext({'a': {'b': 'c', 'pie': 3, 'pi': 3, 'const': 'constant'},
1215  'f': {'g': 'h', 'pie': '3', 'pi': 3, 'const': 'constant'}}),
1216  pyConfigContext({'math': {'pi': 3.14159, 'pie': 3, 'const': 'constant'}}),
1217  pyConfigContext({'geography': {'Europe': 'big', 'const': 'constant'}}),
1218  ))
1219  complicated_context = ExpressionContext(pyConfigContext({'a': {'b': {'pie': 3}}}),)
1220  argcontext = ExpressionContext(
1221  pyConfigContext('{"argv": ["command-name-suffix", "thing-one", "thang-two"]}'),)
1222 
1223  assert FOREACH(("EQ(False, $perms.group.write, $perms.other.write)",), lscontext) is True
1224  assert FOREACH(("EQ($pi, 3)",), Pie_context) is False
1225  assert FOREACH(("EQ($pie, 3)",), Pie_context) is None
1226  assert FOREACH(("$a", "EQ($pie, 3)"), complicated_context) is True
1227  assert FOREACH(("$a", "EQ($pie, 3.14159)"), complicated_context) is False
1228  assert FOREACH(("$a", "EQ($pi, 3.14159)"), complicated_context) is None
1229  assert FOREACH(("EQ($const, constant)",), Pie_context) is True
1230  assert GraphNodeExpression.evaluate('EQ($math.pie, 3)', Pie_context) is True
1231  assert FOREACH(("EQ($group, root)",), lscontext) is True
1232  assert FOREACH(("EQ($owner, root)",), lscontext) is True
1233  assert FOREACH(("AND(EQ($owner, root), EQ($group, root))",), lscontext) is True
1234  assert argmatch(('thing-(.*)',), argcontext) == 'one'
1235  assert argmatch(('THING-(.*)','$argv', 'I'), argcontext) == 'one'
1236  assert argmatch(('thang-(.*)',), argcontext) == 'two'
1237  assert argmatch(('THANG-(.*)','$argv', 'I'), argcontext) == 'two'
1238  assert argmatch(('thang-.*',), argcontext) == 'thang-two'
1239  assert argmatch(('THANG-.*','$argv', 'I'), argcontext) == 'thang-two'
1240  print >> sys.stderr, 'Context tests passed.'
1241 
1242  simpletests()
1243  contexttests()
1244  print >> sys.stderr, 'All tests passed.'
def argmatch(args, context)
def flagvalue(args, context)
def NOTIN(args, _context)
def NONEOK(args, _unused_context)
def serviceipport(args, context)
def is_upstartjob(args, context)
def serviceport(args, context)
def get(self, key, alternative=None)
def argequals(args, context)
def bitwiseOR(args, context)
def selectanipport(arg, _context, preferlowestport=True, preferv4=True)
def FOREACH(args, context)
def ATTRSEARCH(args, context)
def serviceip(args, context)
def match(args, _context)
def PAMMODARGS(args, _context)
def __init__(self, objects, prefix=None)
def FINDATTRVALUE(args, _context)
def IGNORE(_ignoreargs, _ignorecontext)
def bitwiseAND(args, context)
def MUST(args, _unused_context)