The Assimilation Project  based on Assimilation version 1.1.7.1474836767
query.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number colorcolumn=100
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Author: Alan Robertson <alanr@unix.sh>
7 # Copyright (C) 2013 - Assimilation Systems Limited
8 #
9 # Free support is available from the Assimilation Project community - http://assimproj.org
10 # Paid support is available from Assimilation Systems Limited - http://assimilationsystems.com
11 #
12 # The Assimilation software is free software: you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation, either version 3 of the License, or
15 # (at your option) any later version.
16 #
17 # The Assimilation software is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
24 #
25 #
26 '''
27 This module provides classes associated with querying - including providing metadata
28 about these queries for the client code.
29 '''
30 import os, sys, re
31 import collections, operator
32 from py2neo import neo4j
33 from graphnodes import GraphNode, RegisterGraphClass
34 from AssimCclasses import pyConfigContext, pyNetAddr
35 from AssimCtypes import ADDR_FAMILY_IPV6, ADDR_FAMILY_IPV4, ADDR_FAMILY_802
36 from assimjson import JSONtree
37 from bestpractices import BestPractices
38 from cmadb import CMAdb
39 from droneinfo import Drone
40 from consts import CMAconsts
41 
42 @RegisterGraphClass
43 class ClientQuery(GraphNode):
44  '''This class defines queries which can be requested from clients (typically JavaScript)
45  The output of all queries is JSON - as filtered by our security mechanism
46  '''
47  node_query_url = "/doquery/GetaNodeById"
48  def __init__(self, queryname, JSON_metadata=None):
49  '''Parameters
50  ----------
51  JSON_metadata - a JSON string containing
52  'querytype': a string denoting our query type - defaults to 'cypher'
53  'cypher': a string containing our cypher query (if a cypher query)
54  'subtype': a string giving the query subtype (if applicable)
55  'descriptions': a dict containing descriptions in various languages
56  'language-key': A locale-language key
57  'short' a short description of this query in 'language-key'
58  'long' a long description of this query in 'language-key'
59  'parameters': a dict containing
60  'name': name of the parameter to the query
61  'type': the type of the parameter one of several noted
62  'min': optional minimum value for 'name'
63  'max': optional maximum value for 'name'
64  'enumlist': array of possible enum values (type = enum only)
65  'lang': language for this description as a dict:
66  'short':a short description of 'name' in language 'lang'
67  'long': a long description of 'name' in language 'lang'
68  'enumlist': dict of explanations for enumlist above
69 
70  Languages are the typical 'en' 'en_us', 'es', 'es_mx', as per the locale-usual
71  Currently-known types are as follows:
72  int - an integer
73  float - an floating point number
74  string - a string
75  ipaddr - an IP address either ipv4 or ipv6 (as a string)
76  macaddr - a 48 or 64 bit MAC address (as a string)
77  bool - a boolean value
78  hostname- a host name (as string - always converted to lower case)
79  dnsname - a DNS name (as a string - always converted to lower case)
80  enum - a finite enumeration of permissible values (case-insensitive)
81  '''
82 
83  GraphNode.__init__(self, domain='metadata')
84  self.queryname = queryname
85  self.JSON_metadata = JSON_metadata
86  self._store = None
87  self._db = None
88  self._queryobj = None
89  if JSON_metadata is None:
90  self._JSON_metadata = None
91  else:
92  self._JSON_metadata = pyConfigContext(JSON_metadata)
93  if self._JSON_metadata is None:
94  raise ValueError('Parameter JSON_metadata is invalid [%s]' % JSON_metadata)
95 
96  @staticmethod
98  'Return our key attributes in order of decreasing significance'
99  return ['queryname']
100 
101  @staticmethod
102  def set_node_query_url(node_query_url):
103  'Set the base URL for the query operation that returns a node by node id'
104  ClientQuery.node_query_url = node_query_url
105 
106  def post_db_init(self):
107  GraphNode.post_db_init(self)
108  if self._JSON_metadata is None:
109  self._JSON_metadata = pyConfigContext(self.JSON_metadata)
110  self.validate_json()
111 
112  def bind_store(self, store):
113  'Connect our query to a database'
114  db = store.db
115  if self._store is not store:
116  self._store = store
117  if self._db is not db:
118  self._db = db
119  self._queryobj = QueryExecutor.construct_query(self._store, self._JSON_metadata)
120  self.validate_json()
121 
122  def parameter_names(self):
123  'Return the parameter names that go with this query'
124  return self._queryobj.parameter_names()
125 
126  def execute(self, executor_context, idsonly=False, expandJSON=False, maxJSON=0, elemsonly=False
127  , **params):
128  'Execute the query and return an iterator that produces sanitized (filtered) results'
129  if self._db is None:
130  raise ValueError('query must be bound to a Store')
131 
132  queryobj = QueryExecutor.construct_query(self._store, self._JSON_metadata)
133  qparams = queryobj.parameter_names()
134  for pname in qparams:
135  if pname not in params:
136  raise ValueError('Required parameter "%s" for %s query is missing'
137  % (pname, self.queryname))
138  for pname in params.keys():
139  if pname not in qparams:
140  raise ValueError('Excess parameter "%s" supplied for %s query'
141  % (pname, self.queryname))
142  fixedparams = self.validate_parameters(params)
143  resultiter = queryobj.result_iterator(fixedparams)
144  return self.filter_json(executor_context, idsonly, expandJSON
145  , maxJSON, resultiter, elemsonly)
146 
147 
148  def supports_cmdline(self, language='en'):
149  'Return True if this query supports command line formatting'
150  meta = self._JSON_metadata
151  return 'cmdline' in meta and language in meta['cmdline']
152 
153  def cmdline_exec(self, executor_context, language='en', fmtstring=None, **params):
154  'Execute the command line version of the query for the specified language'
155  if fmtstring is None:
156  fmtstring = self._JSON_metadata['cmdline'][language]
157  fixedparams = self.validate_parameters(params)
158  for json in self.execute(executor_context, expandJSON=True
159  , maxJSON=5120, elemsonly=True, **fixedparams):
160  obj = pyConfigContext(json)
161  yield ClientQuery._cmdline_substitute(fmtstring, obj)
162 
163  @staticmethod
164  def _cmdline_substitute(fmtstring, queryresult):
165  '''Perform expression substitution for command line queries.
166  'Substitute fields into the command line output'''
167  chunks = fmtstring.split('${')
168  result = chunks[0]
169  for j in range(1, len(chunks)):
170  # Now we split it up into variable-expression, '}' and extrastuff...
171  (variable, extra) = chunks[j].split('}',1)
172  result += str(JSONtree(queryresult.deepget(variable, 'undefined')))
173  result += extra
174  return result
175 
176  def filter_json(self, executor_context, idsonly, expandJSON, maxJSON
177  , resultiter, elemsonly=False):
178  '''Return a sanitized (filtered) JSON stream from the input iterator
179  The idea of the filtering is to enforce security restrictions on which
180  things can be returned and which fields the executor is allowed to view.
181  This is currently completely ignored, and everthing is returned - as is.
182  This function is a generator.
183 
184  parameters
185  ----------
186  executor_context - security context to execute this in
187  ids_only - if True, return only the URL of the objects (via object id)
188  otherwise return the objects themselves
189  resultiter - iterator giving return results for us to filter
190  '''
191  self = self
192 
193  idsonly = idsonly
194  executor_context = executor_context
195  rowdelim = '{"data":[' if not elemsonly else ''
196  rowcount = 0
197  for result in resultiter:
198  # result is a namedtuple
199  rowcount += 1
200  if len(result) == 1:
201  if idsonly:
202  yield '%s"%s/%d"' % (
203  rowdelim
204  , ClientQuery.node_query_url
205  , Store.id(result[0]))
206  else:
207  yield rowdelim + str(JSONtree(result[0], expandJSON=expandJSON
208  , maxJSON=maxJSON))
209  else:
210  delim = rowdelim + '{'
211  row = ''
212  # W0212: Access to a protected member _fields of a client class
213  # No other way to get the list of columns/fields...
214  # OK - there may be another way, but I didn't how to apply what Nigel told me
215  # pylint: disable=W0212
216  for attr in result._fields:
217  value = getattr(result, attr)
218  if idsonly:
219  row += '%s"%s":"%s"' % (
220  delim
221  , attr
222  , Store.id(value))
223 
224  else:
225  row += '%s"%s":%s' % (
226  delim
227  , attr
228  , str(JSONtree(value, expandJSON=expandJSON, maxJSON=maxJSON)))
229  delim = ','
230  yield row + '}'
231  if not elemsonly:
232  rowdelim = ','
233  if not elemsonly:
234  if rowcount == 0:
235  yield '{"data":[]}'
236  else:
237  yield ']}'
238 
239  # R0912: Too many branches; R0914: too many local variables
240  # pylint: disable=R0914,R0912
241  def validate_json(self):
242  '''Validate the JSON metadata for this query - it's complicated!'''
243  queryobj = QueryExecutor.construct_query(self._store, self._JSON_metadata)
244  query_parameter_names = queryobj.parameter_names()
245  if 'parameters' not in self._JSON_metadata:
246  raise ValueError('parameters missing from metadata')
247  paramdict = self._JSON_metadata['parameters']
248  if 'descriptions' not in self._JSON_metadata:
249  print >> sys.stderr, 'METADATA:', self._JSON_metadata
250  raise ValueError('descriptions missing from metadata')
251 
252  # Validate query descriptions
253  languages = self._JSON_metadata['descriptions']
254  for lang in languages:
255  thislang = languages[lang]
256  if 'short' not in thislang:
257  raise ValueError('"short" query description missing from language %s' % lang)
258  if 'long' not in thislang:
259  raise ValueError('"long" query description missing from language %s' % lang)
260  if 'en' not in languages:
261  raise ValueError("Query description must include language en'")
262  for name in query_parameter_names:
263  if name not in paramdict:
264  raise ValueError('Required parameter %s missing from JSON parameters' % name)
265  for name in paramdict.keys():
266  if name not in query_parameter_names:
267  raise ValueError('JSON parameter %s not required by query' % name)
268 
269  # Validate query parameters
270  for param in queryobj.parameter_names():
271  pinfo = paramdict[param]
272  ptype = pinfo['type']
273  self.validate_query_parameter_metadata(param, pinfo)
274  # Validate parameter information for this (param) language
275  if 'lang' not in pinfo:
276  raise ValueError("Parameter %s must include 'lang' information" % param)
277  langs = pinfo['lang']
278  for lang in languages.keys():
279  if lang not in langs:
280  raise ValueError("Language %s missing from parameter %s" % param)
281  for lang in langs.keys():
282  if lang not in languages:
283  raise ValueError("Language %s missing from query description %s" % lang)
284  for eachlang in langs.keys():
285  thislang = langs[eachlang]
286  if 'short' not in thislang:
287  raise ValueError("Parameter %s, language %s must include 'short' info"
288  % (param, eachlang))
289  if 'long' not in thislang:
290  raise ValueError("Parameter %s, language %s must include 'long' info"
291  % (param, eachlang))
292  if ptype == 'enum' or (ptype == 'list' and pinfo['listtype']['type'] == 'enum'):
293  if 'enumlist' not in thislang:
294  raise ValueError("Parameter %s, language %s must include 'enumlist' info"
295  % (param, eachlang))
296  enums = thislang['enumlist']
297  elist = pinfo['enumlist'] if ptype == 'enum' else pinfo['listtype']['enumlist']
298  for e in elist:
299  if e not in enums:
300  raise ValueError("Parameter %s, language %s missing enum value %s"
301  % (param, eachlang, e))
302  return True
303 
304 
305  def validate_query_parameter_metadata(self, param, pinfo):
306  'Validate the paramater metadata for this query'
307  if 'type' not in pinfo:
308  raise ValueError('Parameter %s missing type field' % param)
309  ptype = pinfo['type']
310  if ptype not in ClientQuery._validationmethods:
311  raise ValueError('Parameter %s has invalid type %s'% (param, ptype))
312  if 'min' in pinfo and ptype != 'int' and ptype != 'float':
313  raise ValueError('Min only valid on numeric fields [%s]'% param )
314  if 'max' in pinfo and ptype != 'int' and ptype != 'float':
315  raise ValueError('Max only valid on numeric fields [%s]'% param )
316  if ptype == 'list':
317  if 'listtype' not in pinfo:
318  raise ValueError('List type [%s] requires listtype'% (param))
319  self.validate_query_parameter_metadata('list', pinfo['listtype'])
320  if ptype == 'enum':
321  if 'enumlist' not in pinfo:
322  raise ValueError('Enum type [%s] requires enumlist'% (param))
323  elist = pinfo['enumlist']
324  for enum in elist:
325  if not isinstance(enum, str) and not isinstance(enum, unicode):
326  raise ValueError('Enumlist values [%s] must be strings - not %s'
327  % (enum, type(enum)))
328 
329  def validate_parameters(self, parameters):
330  '''
331  parameters is a Dict-like object containing parameter names and values
332  '''
333  # Let's see if all the parameters were supplied
334  paramdict = self._JSON_metadata['parameters']
335  queryobj = QueryExecutor.construct_query(self._store, self._JSON_metadata)
336  for param in queryobj.parameter_names():
337  if param not in parameters:
338  raise ValueError('Parameter %s not supplied' % param)
339  # Let's see if any extraneous parameters were supplied
340  for param in parameters.keys():
341  if param not in paramdict:
342  raise ValueError('Invalid Parameter %s supplied' % param)
343  result = {}
344  for param in parameters.keys():
345  value = parameters[param]
346  canonvalue = ClientQuery._validate_value(param, paramdict[param], value)
347  result[param] = canonvalue
348  return result
349 
350 
351 
352  @staticmethod
353  def _validate_int(name, paraminfo, value):
354  'Validate an int value'
355  val = int(value)
356  if 'min' in paraminfo:
357  minval = paraminfo['min']
358  if val < minval:
359  raise ValueError('Value of %s [%s] smaller than mininum [%s]'
360  % (name, val, minval))
361  if 'max' in paraminfo:
362  maxval = paraminfo['max']
363  if val > maxval:
364  raise ValueError('Value of %s [%s] larger than maximum [%s]'
365  % (name, val, maxval))
366  return val
367 
368  @staticmethod
369  def _validate_float(name, paraminfo, value):
370  'Validate an floating point value'
371  val = float(value)
372  if 'min' in paraminfo:
373  minval = paraminfo['min']
374  if val < minval:
375  raise ValueError('Value of %s[%s] smaller than mininum [%s]'
376  % (name, val, minval))
377  if 'max' in paraminfo:
378  maxval = paraminfo['max']
379  if val > maxval:
380  raise ValueError('Value of %s [%s] larger than maximum [%s]'
381  % (name, val, maxval))
382  return val
383 
384  @staticmethod
385  def _validate_string(_name, _paraminfo, value):
386  'Validate a string value (FIXME: should this always be valid??)'
387  # FIXME: This should probably make sure no " or ' or [], ;'s - maybe others?
388  # Probably should allow ":"
389  return value
390 
391  @staticmethod
392  def _validate_macaddr(name, _paraminfo, value):
393  'Validate an MAC address value'
394  mac = pyNetAddr(value)
395  if mac is None:
396  raise ValueError('value of %s [%s] not a valid MAC address' % (name, value))
397  if mac.addrtype() != ADDR_FAMILY_802:
398  raise ValueError('Value of %s [%s] not a MAC address' % (name, value))
399  return str(mac)
400 
401  @staticmethod
402  def _validate_ipaddr(name, _paraminfo, value):
403  'Validate an IP address value'
404  ip = pyNetAddr(value)
405  if ip is None:
406  raise ValueError('Value of %s [%s] not a valid IP address' % (name, value))
407  ip.setport(0)
408  if ip.addrtype() == ADDR_FAMILY_IPV6:
409  return str(ip)
410  if ip.addrtype() == ADDR_FAMILY_IPV4:
411  return str(ip.toIPv6())
412  raise ValueError('Value of %s [%s] not an IP address' % (name, value))
413 
414  @staticmethod
415  def _validate_bool(name, _paraminfo, value):
416  'Validate an Boolean value'
417  if not isinstance(value, bool):
418  raise ValueError('Value of %s [%s] not a boolean' % (name, value))
419  return value
420 
421  @staticmethod
422  def _validate_regex(name, _paraminfo, value):
423  'Validate a regular expression'
424  try:
425  re.compile(value)
426  except re.error as e:
427  raise ValueError('Value of %s ("%s") is not a valid regular expression [%s]' %
428  (name, value, str(e)))
429  return value
430 
431  @staticmethod
432  def _validate_hostname(name, paraminfo, value):
433  'Validate a hostname value'
434  return ClientQuery._validate_dnsname(name, paraminfo, value)
435 
436  @staticmethod
437  def _validate_dnsname(_name, _paraminfo, value):
438  'Validate an DNS name value'
439  value = str(value)
440  return value.lower()
441 
442  @staticmethod
443  def _validate_enum(name, paraminfo, value):
444  'Validate an enumeration value'
445  if 'enumlist' not in paraminfo:
446  raise TypeError("No 'enumlist' for parameter" % name)
447  value = value.tolower()
448  for val in paraminfo['enumlist']:
449  cmpval = val.lower()
450  if cmpval == value:
451  return cmpval
452  raise ValueError('Value of %s [%s] not in enumlist' % (paraminfo['name'], value))
453 
454  @staticmethod
455  def _validate_list(name, paraminfo, listvalue):
456  'Validate a list value'
457  if isinstance(listvalue, (str, unicode)):
458  listvalue = listvalue.split(',')
459  result = []
460  listtype = paraminfo['listtype']
461  for elem in listvalue:
462  result.append(ClientQuery._validate_value(name, listtype, elem))
463  return result
464 
465  @staticmethod
466  def _get_nodetype(nodetype):
467  'Return the value of a node type - if valid'
468  nodetypes = set()
469  for attr in dir(CMAconsts):
470  if attr.startswith('NODE_') and isinstance(getattr(CMAconsts, attr), (str, unicode)):
471  nodetypes.add(attr[5:])
472  nodetypes.add(getattr(CMAconsts, attr))
473  if nodetype not in nodetypes:
474  return None
475  defname = 'NODE_' + nodetype
476  return getattr(CMAconsts, defname) if hasattr(CMAconsts, defname) else nodetype
477 
478 
479  @staticmethod
480  def _validate_nodetype(name, _paraminfo, value):
481  'validate a node type - ignoring case'
482  ret = ClientQuery._get_nodetype(value)
483  if ret is not None:
484  return ret
485  raise ValueError('Value of %s [%s] is not a known node type' % (name, value))
486 
487 
488  @staticmethod
489  def _get_reltype(reltype):
490  'Return the value of a relationship type - if valid'
491  reltypes = set()
492  for attr in dir(CMAconsts):
493  if attr.startswith('REL_') and isinstance(getattr(CMAconsts, attr), (str, unicode)):
494  reltypes.add(attr[4:])
495  reltypes.add(getattr(CMAconsts, attr))
496  if reltype not in reltypes:
497  return None
498  defname = 'REL_' + reltype
499  return getattr(CMAconsts, defname) if hasattr(CMAconsts, defname) else reltype
500 
501  @staticmethod
502  def _validate_reltype(name, _paraminfo, value):
503  'Validate a relationship type - ignoring case'
504  ret = ClientQuery._get_reltype(value)
505  if ret is not None:
506  return ret
507  raise ValueError('Value of %s [%s] is not a known relationship type' % (name, value))
508 
509  _validationmethods = {}
510 
511  @staticmethod
512  def _validate_value(name, paraminfo, value):
513  '''Validate the value given our metadata'''
514  valtype = paraminfo['type']
515  return ClientQuery._validationmethods[valtype](name, paraminfo, value)
516 
517  @staticmethod
518  def load_from_file(store, pathname, queryname=None):
519  'Load a query with metadata from a file'
520  fd = open(pathname, 'r')
521  json = fd.read()
522  fd.close()
523  if pyConfigContext(json) is None:
524  raise ValueError ('ERROR: Contents of %s is not valid JSON.' % (pathname))
525  if queryname is None:
526  queryname = os.path.basename(pathname)
527  #print 'LOADING %s as %s' % (pathname, queryname)
528  ret = store.load_or_create(ClientQuery, queryname=queryname, JSON_metadata=json)
529  ret.JSON_metadata = json
530  ret.bind_store(store)
531  if not ret.validate_json():
532  print >> sys.stderr, ('ERROR: Contents of %s is not a valid query.' % pathname)
533 
534  return ret
535 
536  @staticmethod
537  def load_directory(store, directoryname):
538  'Returns a generator that returns all the Queries in that directory'
539  files = os.listdir(directoryname)
540  files.sort()
541  for filename in files:
542  path = os.path.join(directoryname, filename)
543  try:
544  yield ClientQuery.load_from_file(store, path)
545  except ValueError as e:
546  print >> sys.stderr, 'File %s is invalid: %s' % (path, str(e))
547 
548  @staticmethod
549  def load_tree(store, rootdirname, followlinks=False):
550  'Returns a generator that will returns all the Queries in that directory structure'
551  tree = os.walk(rootdirname, topdown=True, onerror=None, followlinks=followlinks)
552  rootprefixlen = len(rootdirname)+1
553  for walktuple in tree:
554  (dirpath, dirnames, filenames) = walktuple
555  dirnames.sort()
556  prefix = dirpath[rootprefixlen:]
557  filenames.sort()
558  for filename in filenames:
559  queryname = prefix + filename
560  path = os.path.join(dirpath, filename)
561  if filename.startswith('.'):
562  continue
563  try:
564  yield ClientQuery.load_from_file(store, path, queryname)
565  except ValueError as e:
566  print >> sys.stderr, 'File %s is invalid: %s' % (path, str(e))
567 
568 class QueryExecutor(object):
569  '''An abstract class which knows which can perform a variety of types of queries
570  At the moment that's "python" and "cypher".
571  '''
572  DEFAULT_EXECUTOR_METHOD = 'CypherExecutor'
573  EXECUTOR_METHODS = {}
574 
575  def __init__(self, store, metadata):
576  '''Construct an object remembering our metadata'''
577  self.store = store
578  self.metadata = metadata
579 
580  @staticmethod
581  def construct_query(store, metadata):
582  '''Construct a query of the type requested.
583  We return None if we can't construct a query from our metadata.
584  '''
585  querytype = (metadata['querytype'] if 'querytype' in metadata
586  else QueryExecutor.DEFAULT_EXECUTOR_METHOD)
587 
588  if querytype not in QueryExecutor.EXECUTOR_METHODS:
589  raise ValueError('Querytype %s is not a valid query type' % querytype)
590  queryclass = QueryExecutor.EXECUTOR_METHODS[querytype]
591  return (queryclass.construct_query(store, metadata)
592  if querytype in QueryExecutor.EXECUTOR_METHODS else None)
593 
594  @staticmethod
595  def register(ourclass):
596  'Register this class as a QueryExecutor subclass'
597  QueryExecutor.EXECUTOR_METHODS[ourclass.__name__] = ourclass
598  return ourclass
599 
600  def parameter_names(self):
601  '''We return a set of parameters that we expect.
602  We return None if we are flexible (or don't know) about our expected parameters.
603  '''
604  raise NotImplementedError('QueryExecutor is an abstract class')
605 
606  def result_iterator(self, params):
607  '''We return an iterator which will yield the results of performing
608  this query with these parameters.
609  '''
610  raise NotImplementedError('QueryExecutor is an abstract class')
611 
612 
613 @QueryExecutor.register
615  '''QueryExecutor subclass for Cypher queries'''
616 
617  @staticmethod
618  def construct_query(store, metadata):
619  'Call the CypherExecutor constructor'
620  return CypherExecutor(store, metadata)
621 
622  def __init__(self, store, metadata):
623  if 'cypher' not in metadata:
624  raise ValueError('cypher query missing from metadata: %s' % str(metadata))
625  QueryExecutor.__init__(self, store, metadata)
626  self.query = metadata['cypher']
627 
628  def parameter_names(self):
629  '''We return a set of parameters that we expect.
630  We return None if we are flexible (or don't know) about our expected parameters.
631  Return the parameter names our cypher query uses'''
632 
633  START = 1
634  BACKSLASH = 2
635  GOTLCURLY = 3
636  results = []
637  paramname = ''
638  state = START
639 
640  for c in self.metadata['cypher']:
641  if state == START:
642  if c == '\\':
643  state = BACKSLASH
644  if c == '{':
645  state = GOTLCURLY
646  elif state == BACKSLASH:
647  state = START
648  else: # GOTLCURLY
649  if c == '}':
650  if paramname != '':
651  results.append(paramname)
652  paramname = ''
653  state = START
654  else:
655  paramname += c
656  return results
657 
658  def result_iterator(self, params):
659  '''We return an iterator which will yield the results of performing
660  this query with these parameters.
661  '''
662  return self.store.load_cypher_query(self.query, GraphNode.factory, params=params)
663 
664 @QueryExecutor.register
666  '''QueryExecutor subclass for Python code queries'''
667 
668  EXECUTOR_METHODS = {}
669  PARAMETERS = []
670 
671  def parameter_names(self):
672  '''We return a set of parameters that we expect.
673  We return None if we are flexible (or don't know) about our expected parameters.
674  Return the parameter names our cypher query uses'''
675  return self.PARAMETERS
676 
677  @staticmethod
678  def register(ourclass):
679  PythonExec.EXECUTOR_METHODS[ourclass.__name__] = ourclass
680  return ourclass
681 
682  def result_iterator(self, params):
683  '''We return an iterator which will yield the results of performing
684  this query with these parameters.
685  '''
686  raise NotImplementedError('PythonExec is an abstract class')
687 
688  @staticmethod
689  def construct_query(store, metadata):
690  'Call the subclass constructor'
691  if 'subtype' not in metadata:
692  raise ValueError('subtype missing from PythonExec metadata')
693  subclassname = metadata['subtype']
694  if subclassname not in PythonExec.EXECUTOR_METHODS:
695  raise ValueError('%s is not a valid PythonExec subtype' % subclassname)
696  subclass = PythonExec.EXECUTOR_METHODS[subclassname]
697  return subclass(store, metadata)
698 
699 @PythonExec.register
701  '''Return discovery type+rule scores for all discovery types'''
702  PARAMETERS = []
703 
704  def result_iterator(self, _params):
705  '''We return an iterator which will yield the results of performing
706  this query with these parameters.
707  '''
708  dtype_totals, _drone_totals, rule_totals = grab_category_scores(self.store)
709  # 0: domain
710  # 1: category name
711  # 2: discovery-type
712  # 3: total score for this discovery type _across all rules
713  # 4: rule id
714  # 5: total score for this rule id
715  sortkeys = operator.itemgetter(1,3,5,2,4,0)
716  for tup in sorted(yield_rule_scores([], dtype_totals, rule_totals),
717  key=sortkeys, reverse=True):
718  yield tup
719 
720 @PythonExec.register
722  '''query executor returning discovery type+rule scores for security scores'''
723  PARAMETERS = []
724  def result_iterator(self, _params):
725  '''We return an iterator which will yield the results of performing
726  this query with these parameters.
727  '''
728  dtype_totals, _drone_totals, rule_totals =grab_category_scores(self.store,
729  categories='security')
730  # 0: domain
731  # 1: category name
732  # 2: discovery-type
733  # 3: total score for this discovery type _across all rules
734  # 4: rule id
735  # 5: total score for this rule id
736  sortkeys = operator.itemgetter(1,3,5,2,4,0)
737  for tup in sorted(yield_rule_scores(['security'], dtype_totals, rule_totals),
738  key=sortkeys, reverse=True):
739  yield tup
740 
741 @PythonExec.register
743  '''Return discovery type+host security scores'''
744  PARAMETERS = []
745  PARAMETERS = []
746  def result_iterator(self, _params):
747  dtype_totals, drone_totals, _rule_totals = grab_category_scores(self.store)
748  # 0: domain
749  # 1: category name
750  # 2: discovery-type
751  # 3: total score for this discovery type _across all drones
752  # 4: drone designation (name)
753  # 5: total score for this drone for this discovery type
754  sortkeys = operator.itemgetter(0,3,5,2,4,1)
755  for tup in sorted(yield_drone_scores([], drone_totals, dtype_totals),
756  key=sortkeys, reverse=True):
757  yield tup
758 
759 @PythonExec.register
761  '''query executor returning discovery type+host scores for all score types'''
762  PARAMETERS = []
763  def result_iterator(self, _params):
764  dtype_totals, drone_totals, _rule_totals = grab_category_scores(self.store)
765  # 0: domain
766  # 1: category name
767  # 2: discovery-type
768  # 3: total score for this discovery type _across all drones
769  # 4: drone designation (name)
770  # 5: total score for this drone for this discovery type
771  sortkeys = operator.itemgetter(0,3,5,2,4,1)
772  for tup in sorted(yield_drone_scores([], drone_totals, dtype_totals),
773  key=sortkeys, reverse=True):
774  yield tup
775 @PythonExec.register
777  '''query executor returning domain, score-category, total-score'''
778  PARAMETERS = []
779  def result_iterator(self, _params):
780  dtype_totals, _drone_totals, _rule_totals = grab_category_scores(self.store)
781  for tup in yield_total_scores(dtype_totals):
782  yield tup
783 
784 
785 def setup_dict3(d, key1, key2, key3):
786  'Initialize the given subkey (3 layers down)to 0.0'
787  if key1 not in d:
788  d[key1] = {}
789  if key2 not in d[key1]:
790  d[key1][key2] = {}
791  if key3 not in d[key1][key2]:
792  d[key1][key2][key3] = 0.0
793 
794 def setup_dict4(d, key1, key2, key3, key4):
795  'Initialize the given subkey (4 layers down)to 0.0'
796  if key1 not in d:
797  d[key1] = {}
798  if key2 not in d[key1]:
799  d[key1][key2] = {}
800  if key3 not in d[key1][key2]:
801  d[key1][key2][key3] = {}
802  if key4 not in d[key1][key2][key3]:
803  d[key1][key2][key3][key4] = 0.0
804 
805 
806 # [R0914:grab_category_scores] Too many local variables (19/15)
807 # pylint: disable=R0914
808 def grab_category_scores(store, categories=None, domains=None, debug=False):
809  '''Method to create and return some python Dicts with security scores and totals by category
810  and totals by drone/category
811  Categories is None, a desired category, or a list of desired categories.
812  domains is None, a desired domain, or a list of desired domains.
813  '''
814  if domains is None:
815  cypher = '''START drone=node:Drone('*:*') RETURN drone'''
816  else:
817  domains = (domains,) if isinstance(domains, (str, unicode)) else list(domains)
818  cypher = ("START drone=node:Drone('*:*') WHERE drone.domain IN %s RETURN drone"
819  % str(list(domains)))
820  if categories is not None:
821  categories = (categories,) if isinstance(categories, (str, unicode)) else list(categories)
822 
823  bpobj = BestPractices(CMAdb.io.config, CMAdb.io, store, CMAdb.log, debug=debug)
824  dtype_totals = {} # scores organized by (domain, category, discovery-type)
825  drone_totals = {} # scores organized by (domain, category, discovery-type, drone)
826  rule_totals = {} # scores organized by (domain, category, discovery-type, rule)
827 
828  for drone in store.load_cypher_nodes(cypher, Drone):
829  domain = drone.domain
830  designation = drone.designation
831  discoverytypes = drone.bp_discoverytypes_list()
832  for dtype in discoverytypes:
833  dattr = Drone.bp_discoverytype_result_attrname(dtype)
834  statuses = getattr(drone, dattr)
835  for rule_obj in BestPractices.eval_objects[dtype]:
836  rulesobj = rule_obj.fetch_rules(drone, None, dtype)
837  _, scores, rulescores = bpobj.compute_scores(drone, rulesobj, statuses)
838  for category in scores:
839  if categories and category not in categories:
840  continue
841  # Accumulate scores by (domain, category, discovery_type)
842  setup_dict3(dtype_totals, domain, category, dtype)
843  dtype_totals[domain][category][dtype] += scores[category]
844  # Accumulate scores by (domain, category, discovery_type, drone)
845  setup_dict4(drone_totals, domain, category, dtype, designation)
846  drone_totals[domain][category][dtype][designation] += scores[category]
847  # Accumulate scores by (domain, category, discovery_type, ruleid)
848  for ruleid in rulescores[category]:
849  setup_dict4(rule_totals, domain, category, dtype, ruleid)
850  rule_totals[domain][category][dtype][ruleid] += rulescores[category][ruleid]
851 
852  return dtype_totals, drone_totals, rule_totals
853 
854 def yield_total_scores(dtype_totals, categories=None):
855  '''Format the total scores by category as a named tuple.
856  We output the following fields:
857  0: domain
858  1: category name
859  3: total score for this category
860  '''
861  TotalScore = collections.namedtuple('TotalScore', ['domain', 'category', 'score'])
862  for domain in sorted(dtype_totals):
863  domain_scores = dtype_totals[domain]
864  for category in sorted(domain_scores):
865  total = 0.0
866  if categories is not None and category not in categories:
867  continue
868  cat_scores = domain_scores[category]
869  for dtype in cat_scores:
870  total += cat_scores[dtype]
871  yield TotalScore(domain, category, total)
872 
873 def yield_drone_scores(categories, drone_totals, dtype_totals):
874  '''Format the drone_totals + dtype_totals as a named tuple
875  We output the following fields:
876  0: domain
877  1: category name
878  2: discovery-type
879  3: total score for this discovery type _across all drones_
880  4: drone designation (name)
881  5: total score for this drone for this discovery type
882  '''
883  DroneScore = collections.namedtuple('DroneScore', ['domain', 'category', 'discovery_type',
884  'dtype_score', 'drone', 'drone_score'])
885  for domain in drone_totals:
886  for cat in drone_totals[domain]:
887  if categories and cat not in categories:
888  continue
889  for dtype in drone_totals[domain][cat]:
890  for drone in drone_totals[domain][cat][dtype]:
891  score = drone_totals[domain][cat][dtype][drone]
892  if score > 0:
893  yield DroneScore(domain, cat, dtype, dtype_totals[domain][cat][dtype],
894  drone, score)
895 
896 def yield_rule_scores(categories, dtype_totals, rule_totals):
897  '''Format the rule totals + dtype_totals as a CSV-style output
898  We output the following fields:
899  0: domain
900  1: category name
901  2: discovery-type
902  3: total score for this discovery type _across all rules
903  5: rule id
904  6: total score for this rule id
905  '''
906  # rule_totals = # scores organized by (category, discovery-type, rule)
907 
908  RuleScore = collections.namedtuple('RuleScore',
909  ['domain', 'category', 'discovery_type', 'dtype_score',
910  'ruleid', 'ruleid_score'])
911  for domain in rule_totals:
912  for cat in rule_totals[domain]:
913  if categories and cat not in categories:
914  continue
915  for dtype in rule_totals[domain][cat]:
916  for ruleid in rule_totals[domain][cat][dtype]:
917  score = rule_totals[domain][cat][dtype][ruleid]
918  if score > 0:
919  yield RuleScore(domain, cat, dtype, dtype_totals[domain][cat][dtype],
920  ruleid, score)
921 PackageTuple = collections.namedtuple('PackageTuple',
922  ['domain', 'drone', 'package', 'version', 'packagetype'])
923 @PythonExec.register
925  '''query executor returning packages matching the given prefix'''
926  PARAMETERS = ['prefix']
927  def result_iterator(self, params):
928  prefix = params['prefix']
929  # 0: domain
930  # 1: Drone
931  # 2: Package name
932  # 3: Package Version
933  # 4: Package type
934  cypher = (
935  '''MATCH (system)-[rel:jsonattr]->(jsonmap)
936  WHERE system.nodetype in ['Drone', 'DockerSystem', 'VagrantSystem']
937  AND jsonmap.nodetype = 'JSONMapNode'
938  AND rel.jsonname =~ '^_init_packages.*' AND jsonmap.json CONTAINS '"%s'
939  RETURN system, jsonmap.json AS json ORDER BY system.domain, system.designation
940  ''' % prefix)
941  for (drone, json) in self.store.load_cypher_query(cypher, Drone):
942  jsonobj = pyConfigContext(json)
943  # pylint is confused here - jsonobj['data'] _is_ very much iterable...
944  # pylint: disable=E1133
945  jsondata = jsonobj['data']
946  for pkgtype in jsondata:
947  for package in jsondata[pkgtype]:
948  if package.startswith(prefix):
949  yield PackageTuple(drone.domain, drone, package,
950  jsondata[pkgtype][package], pkgtype)
951 
952 
953 @PythonExec.register
955  '''query executor returning all packages on all systems'''
956  PARAMETERS = []
957  def result_iterator(self, params):
958  # 0: domain
959  # 1: Drone
960  # 2: Package name
961  # 3: Package Version
962  cypher = (
963  '''MATCH (system)-[rel:jsonattr]->(jsonmap)
964  WHERE system.nodetype in ['Drone', 'DockerSystem', 'VagrantSystem']
965  AND rel.jsonname =~ '^_init_packages.*'
966  RETURN system, jsonmap.json AS json ORDER BY system.domain, system.designation
967  ''')
968 
969  for (drone, json) in self.store.load_cypher_query(cypher, GraphNode.factory):
970  jsonobj = pyConfigContext(json)
971  # pylint is confused here - jsonobj['data'] _is_ very much iterable...
972  # pylint: disable=E1133
973  jsondata = jsonobj['data']
974  for pkgtype in jsondata:
975  for package in jsondata[pkgtype]:
976  yield PackageTuple(drone.domain, drone, package,
977  jsondata[pkgtype][package], pkgtype)
978 
979 @PythonExec.register
981  '''query executor returning packages matching the given regular expression'''
982  PARAMETERS = ['regex']
983  def result_iterator(self, params):
984  regex = params['regex']
985  # 0: domain
986  # 1: Drone
987  # 2: Package name
988  # 3: Package Version
989  cypher = (
990  '''START drone=node:Drone('*:*')
991  MATCH (drone)-[rel:jsonattr]->(jsonmap)
992  WHERE rel.jsonname =~ '^_init_packages.*' AND jsonmap.json =~ '.*%s.*.*'
993  RETURN drone, jsonmap.json AS json ORDER BY system.domain, system.designation
994  ''' % regex)
995 
996  regexobj = re.compile('.*' + regex)
997  for (drone, json) in self.store.load_cypher_query(cypher, GraphNode.factory):
998  jsonobj = pyConfigContext(json)
999  # pylint is confused here - jsonobj['data'] _is_ very much iterable...
1000  # pylint: disable=E1133
1001  jsondata = jsonobj['data']
1002  for pkgtype in jsondata:
1003  for package in jsondata[pkgtype]:
1004  if regexobj.match(package):
1005  yield PackageTuple(drone.domain, drone, package,
1006  jsondata[pkgtype][package], pkgtype)
1007 
1008 
1009 @PythonExec.register
1011  '''query executor returning packages of the given name'''
1012  PARAMETERS = ['packagename']
1013  def result_iterator(self, params):
1014  packagename = params['packagename']
1015  if packagename.find('::') < 0:
1016  packagename += '::'
1017  # 0: domain
1018  # 1: Drone
1019  # 2: Package name
1020  # 3: Package Version
1021  cypher = (
1022  '''START drone=node:Drone('*:*')
1023  MATCH (drone)-[rel:jsonattr]->(jsonmap)
1024  WHERE rel.jsonname = '_init_packages' AND jsonmap.json CONTAINS '"%s'
1025  return drone, jsonmap.json as json
1026  ''' % packagename)
1027  for (drone, json) in self.store.load_cypher_query(cypher, GraphNode.factory):
1028  jsonobj = pyConfigContext(json)
1029  # pylint is confused here - jsonobj['data'] _is_ very much iterable...
1030  # pylint: disable=E1133
1031  jsondata = jsonobj['data']
1032  for pkgtype in jsondata:
1033  for package in jsondata[pkgtype]:
1034  if package.startswith(packagename):
1035  yield PackageTuple(drone.domain, drone, package,
1036  jsondata[pkgtype][package], pkgtype)
1037 
1038 def reltype_expr(reltypes):
1039  'Create a Cypher query expression for (multiple) relationship types'
1040  if isinstance(reltypes, (str, unicode)):
1041  reltypes = (reltypes,)
1042  relationship_expression = ''
1043  delim=''
1044  for reltype in reltypes:
1045  relationship_expression += '%s:%s' % (delim, reltype)
1046  delim = '|'
1047  return relationship_expression
1048 
1049 @PythonExec.register
1051  'A class to return a subgraph centered around one or more Drones'
1052  PARAMETERS = ['nodetypes', 'reltypes', 'hostname']
1053  basequery = \
1054  '''START start=node:Drone('*:*')
1055  WHERE start.nodetype = 'Drone' AND start.designation in '%s'
1056  MATCH p = shortestPath( (start)-[%s*]-(m) )
1057  WHERE m.nodetype IN %s
1058  UNWIND nodes(p) AS n
1059  UNWIND rels(p) AS r
1060  RETURN [x in COLLECT(DISTINCT n) WHERE x.nodetype in %s] AS nodes,
1061  COLLECT(DISTINCT r) AS relationships'''
1062 
1063  def result_iterator(self, params):
1064  nodetypes = params['nodetypes']
1065  reltypes = params['reltypes']
1066  designation = params['hostname']
1067  if isinstance(designation, (str, unicode)):
1068  designation = [designation]
1069  designation_s = str(designation)
1070  relstr = reltype_expr(reltypes)
1071  nodestr = str(nodetypes)
1072  query = PythonDroneSubgraphQuery.basequery % (designation_s, relstr, nodestr, nodestr)
1073  #print >> sys.stderr, 'RUNNING THIS QUERY:', query
1074  for row in self.store.load_cypher_query(query, GraphNode.factory):
1075  yield row
1076 
1077 @PythonExec.register
1079  'A class to return a subgraph centered around a Drone'
1080  PARAMETERS = ['nodetypes', 'reltypes']
1081  basequery = \
1082  '''START start=node:Drone('*:*')
1083  WHERE start.nodetype = 'Drone'
1084  MATCH p = shortestPath( (start)-[%s*]-(m) )
1085  WHERE m.nodetype IN %s
1086  UNWIND nodes(p) AS n
1087  UNWIND rels(p) AS r
1088  RETURN [x in COLLECT(DISTINCT n) WHERE x.nodetype in %s] AS nodes,
1089  COLLECT(DISTINCT r) AS relationships'''
1090 
1091  def result_iterator(self, params):
1092  nodetypes = params['nodetypes']
1093  reltypes = params['reltypes']
1094  relstr = reltype_expr(reltypes)
1095  nodestr = str(nodetypes)
1096  query = PythonAllDronesSubgraphQuery.basequery % (relstr, nodestr, nodestr)
1097  #print >> sys.stderr, 'RUNNING THIS QUERY:', query
1098  for row in self.store.load_cypher_query(query, GraphNode.factory):
1099  yield row
1100 
1101 
1102 # message W0212: access to protected member of client class
1103 # pylint: disable=W0212
1104 ClientQuery._validationmethods = {
1105  'int': ClientQuery._validate_int,
1106  'float': ClientQuery._validate_float,
1107  'bool': ClientQuery._validate_bool,
1108  'string': ClientQuery._validate_string,
1109  'enum': ClientQuery._validate_enum,
1110  'ipaddr': ClientQuery._validate_ipaddr,
1111  'list': ClientQuery._validate_list,
1112  'macaddr': ClientQuery._validate_macaddr,
1113  'hostname': ClientQuery._validate_hostname,
1114  'dnsname': ClientQuery._validate_dnsname,
1115  'regex': ClientQuery._validate_regex,
1116  'nodetype': ClientQuery._validate_nodetype,
1117  'reltype': ClientQuery._validate_reltype,
1118 }
1119 
1120 if __name__ == '__main__':
1121  # pylint: disable=C0413
1122  from store import Store
1123  from cmadb import Neo4jCreds
1124  metadata1 = \
1125  '''
1126  { "cypher": "START n=node:ClientQuery('*:*') RETURN n",
1127  "parameters": {},
1128  "descriptions": {
1129  "en": {
1130  "short": "list all queries",
1131  "long": "return a list of all available queries"
1132  }
1133  }
1134  }
1135  '''
1136  q1 = ClientQuery('allqueries', metadata1)
1137 
1138  metadata2 = \
1139  '''
1140  { "cypher": "START n=node:ClientQuery('{queryname}:metadata') RETURN n",
1141  "parameters": {
1142  "queryname": {
1143  "type": "string",
1144  "lang": {
1145  "en": {
1146  "short": "query name",
1147  "long": "Name of query to retrieve"
1148  }
1149  }
1150  }
1151  },
1152  "descriptions": {
1153  "en": {
1154  "short": "Retrieve a query",
1155  "long": "Retrieve all the information about a query"
1156  }
1157  }
1158  }
1159  '''
1160  q2 = ClientQuery('allqueries', metadata2)
1161 
1162  metadata3 = \
1163  '''
1164  {
1165  "cypher": "START ip=node:IPaddr('{ipaddr}:*')
1166  MATCH (ip)<-[:ipowner]-()<-[:nicowner]-(system)
1167  RETURN system",
1168 
1169  "descriptions": {
1170  "en": {
1171  "short": "get system from IP",
1172  "long": "retrieve the system owning the requested IP"
1173  }
1174  },
1175  "parameters": {
1176  "ipaddr": {
1177  "type": "ipaddr",
1178  "lang": {
1179  "en": {
1180  "short": "IP address",
1181  "long": "IP (IPv4 or IPv6) address of system of interest"
1182  }
1183  }
1184  }
1185  }
1186  }
1187  '''
1188  metadata4 = \
1189  r''' {
1190  "cypher": "START start=node:Drone('*:*')
1191  WHERE start.nodetype = 'Drone' AND start.designation = '{host}'
1192  MATCH p = shortestPath( (start)-[*]-(m) )
1193  WHERE m.nodetype IN {nodetypes}
1194  UNWIND nodes(p) as n
1195  UNWIND rels(p) as r
1196  RETURN [x in collect(distinct n) WHERE x.nodetype in {nodetypes}]] as nodes,
1197  collect(distinct r) as relationships",
1198  "copyright": "Copyright(C) 2014 Assimilation Systems Limited",
1199  "descriptions": {
1200  "en": {
1201  "short": "return entire graph",
1202  "long": "retrieve all nodes and all relationships"
1203  }
1204  },
1205  "parameters": {
1206  "host": {
1207  "type": "hostname",
1208  "lang": {
1209  "en": {
1210  "short": "starting host name",
1211  "long": "name of host to start the query at"
1212  }
1213  }
1214  },
1215  "nodetypes": {
1216  "type": "list",
1217  "listtype": {
1218  "type": "nodetype"
1219  },
1220  "lang": {
1221  "en": {
1222  "short": "node types",
1223  "long": "set of node types to include in query result",
1224  }
1225  }
1226  }
1227  },
1228  "cmdline": {
1229  "en": "{\"nodes\":${nodes}, \"relationships\": ${relationships}}",
1230  "script": "{\"nodes\":${nodes}, \"relationships\": ${relationships}}"
1231  },
1232  }'''
1233  q3 = ClientQuery('ipowners', metadata3)
1234  q3.validate_json()
1235  q4 = ClientQuery('subgraph', metadata4)
1236  q4.validate_json()
1237 
1238  Neo4jCreds().authenticate()
1239  neodb = neo4j.Graph()
1240  neodb.delete_all()
1241 
1242  umap = {'ClientQuery': True}
1243  ckmap = {'ClientQuery': {'index': 'ClientQuery', 'kattr':'queryname', 'value':'None'}}
1244 
1245  qstore = Store(neodb, uniqueindexmap=umap, classkeymap=ckmap)
1246  for classname in GraphNode.classmap:
1247  GraphNode.initclasstypeobj(qstore, classname)
1248 
1249  print "LOADING TREE!"
1250 
1251  dirname = os.path.dirname(sys.argv[0])
1252  dirname = '.' if dirname == '' else dirname
1253  queries = ClientQuery.load_tree(qstore, "%s/../queries" % dirname)
1254  qlist = [q for q in queries]
1255  qstore.commit()
1256  print "%d node TREE LOADED!" % len(qlist)
1257  qe2 = qstore.load_or_create(ClientQuery, queryname='list')
1258  qe2.bind_store(qstore)
1259  testresult = ''
1260  for s in qe2.execute(None, idsonly=False, expandJSON=True):
1261  testresult += s
1262  print 'RESULT', testresult
1263  # Test out a command line query
1264  for s in qe2.cmdline_exec(None):
1265  if re.match(s, '[ ]unknown$'):
1266  raise RuntimeError('Search result contains unknown: %s' % s)
1267  print s
1268 
1269  print "All done!"
def result_iterator(self, params)
Definition: query.py:957
def load_from_file(store, pathname, queryname=None)
Definition: query.py:518
def result_iterator(self, _params)
Definition: query.py:704
def construct_query(store, metadata)
Definition: query.py:689
def result_iterator(self, params)
Definition: query.py:658
def result_iterator(self, params)
Definition: query.py:1091
def result_iterator(self, _params)
Definition: query.py:746
def parameter_names(self)
Definition: query.py:671
def parameter_names(self)
Definition: query.py:600
def construct_query(store, metadata)
Definition: query.py:581
def grab_category_scores(store, categories=None, domains=None, debug=False)
Definition: query.py:808
def load_directory(store, directoryname)
Definition: query.py:537
def result_iterator(self, params)
Definition: query.py:1063
def result_iterator(self, params)
Definition: query.py:927
def bind_store(self, store)
Definition: query.py:112
def setup_dict4(d, key1, key2, key3, key4)
Definition: query.py:794
def __init__(self, queryname, JSON_metadata=None)
Definition: query.py:48
def register(ourclass)
Definition: query.py:595
def result_iterator(self, params)
Definition: query.py:606
def __meta_keyattrs__()
Definition: query.py:97
def yield_total_scores(dtype_totals, categories=None)
Definition: query.py:854
def filter_json(self, executor_context, idsonly, expandJSON, maxJSON, resultiter, elemsonly=False)
Definition: query.py:177
def result_iterator(self, params)
Definition: query.py:682
def validate_query_parameter_metadata(self, param, pinfo)
Definition: query.py:305
def post_db_init(self)
Definition: query.py:106
def reltype_expr(reltypes)
Definition: query.py:1038
def execute(self, executor_context, idsonly=False, expandJSON=False, maxJSON=0, elemsonly=False, params)
Definition: query.py:127
def result_iterator(self, _params)
Definition: query.py:763
def construct_query(store, metadata)
Definition: query.py:618
def validate_parameters(self, parameters)
Definition: query.py:329
def result_iterator(self, params)
Definition: query.py:1013
def __init__(self, store, metadata)
Definition: query.py:622
def set_node_query_url(node_query_url)
Definition: query.py:102
def yield_rule_scores(categories, dtype_totals, rule_totals)
Definition: query.py:896
def result_iterator(self, params)
Definition: query.py:983
def load_tree(store, rootdirname, followlinks=False)
Definition: query.py:549
def cmdline_exec(self, executor_context, language='en', fmtstring=None, params)
Definition: query.py:153
def __init__(self, store, metadata)
Definition: query.py:575
def parameter_names(self)
Definition: query.py:628
def supports_cmdline(self, language='en')
Definition: query.py:148
def result_iterator(self, _params)
Definition: query.py:724
def validate_json(self)
Definition: query.py:241
def yield_drone_scores(categories, drone_totals, dtype_totals)
Definition: query.py:873
def register(ourclass)
Definition: query.py:678
def setup_dict3(d, key1, key2, key3)
Definition: query.py:785
PackageTuple
Definition: query.py:921
def result_iterator(self, _params)
Definition: query.py:779
def parameter_names(self)
Definition: query.py:122