The Assimilation Project  based on Assimilation version 1.1.7.1474836767
arpdiscovery.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # vim: smartindent tabstop=4 shiftwidth=4 expandtab number colorcolumn=100
3 #
4 # This file is part of the Assimilation Project.
5 #
6 # Author: Alan Robertson <alanr@unix.sh>
7 # Copyright (C) 2014 - Assimilation Systems Limited
8 #
9 # Free support is available from the Assimilation Project community - http://assimproj.org
10 # Paid support is available from Assimilation Systems Limited - http://assimilationsystems.com
11 #
12 # The Assimilation software is free software: you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation, either version 3 of the License, or
15 # (at your option) any later version.
16 #
17 # The Assimilation software is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
24 #
25 #
26 
27 '''
28 ARP Discovery Listener code.
29 This is the class for handling ARP discovery packets as they arrive.
30 These packets include data from all nanoprobe-observed ARP packets - broadcast and unicast.
31 It is a subclass of the DiscoveryListener class.
32 
33 More details are documented in the ArpDiscoveryListener class
34 '''
35 
36 import sys
37 from consts import CMAconsts
38 from store import Store
39 from AssimCclasses import pyConfigContext
40 from AssimCtypes import CONFIGNAME_INSTANCE, CONFIGNAME_DEVNAME
41 from discoverylistener import DiscoveryListener
42 from graphnodes import NICNode, IPaddrNode, GraphNode
43 from systemnode import SystemNode
44 from cmaconfig import ConfigFile
45 from linkdiscovery import discovery_indicates_link_is_up
46 
47 @SystemNode.add_json_processor # Register ourselves to process discovery packets
48 class ArpDiscoveryListener(DiscoveryListener):
49  '''Class for processing ARP cache discovery entries.
50  The data section contains (IPaddress, MACaddress) pairs as a hash table (JSON object)
51  Nanoprobes currently send their entire cache each time anything shows up
52 
53  For interest, here are some default ARP cache timeouts as of this writing:
54  Linux 300 seconds
55  Solaris 300 seconds
56  Windows 600 seconds
57  AIX 1200 seconds
58  FreeBSD 1200 seconds
59  NetBSD 1200 seconds
60  OpenBSD 1200 seconds
61  VMWare 1200 seconds
62  Cisco 14400 seconds
63 
64  For large subnets, it would be much more efficient to send only changes --
65  additions and deletions -- but that's not what we currently do :-D
66 
67  @TODO: Change ARP updates to give deltas
68 
69  If we changed the nanoprobes to send delta updates, then you wouldn't particularly
70  want a 20 minute timeout - you'd want something more like a few minutes
71  so that they came a few at a time to the CMA. This would keep the CMA from
72  being bottlenecked by massive updates if you have a subnet with a lot of IP
73  addresses on it. It also means that we would not get 1024 entries just because
74  one came online...
75 
76  Of course, a lot of what causes this code to be really slow is the fact that we
77  hit the database with a transaction for each IP and each MAC that we find in the
78  message.
79  '''
80 
81  prio = DiscoveryListener.PRI_OPTION # This is an optional feature
82  # We are interested in two kinds of packets:
83  # netconfig: Packets from network configuration discovery
84  # When we hear these we send requests to listen to ARPs
85  # This is what eventually causes ARP discovery packets to be sent
86  # ARP: Packets resulting from ARP discovery - triggered by
87  # the requests we send above...
88  wantedpackets = ('ARP', 'netconfig')
89 
90  ip_map = {}
91  mac_map = {}
92 
93  def processpkt(self, drone, srcaddr, jsonobj, discoverychanged):
94  '''Trigger ARP discovery or add ARP data to the database.
95  '''
96  if not discoverychanged:
97  return
98  if jsonobj['discovertype'] == 'ARP':
99  self.processpkt_arp(drone, srcaddr, jsonobj)
100  elif jsonobj['discovertype'] == 'netconfig':
101  self.processpkt_netconfig(drone, srcaddr, jsonobj)
102  else:
103  print >> sys.stderr, 'OOPS! unexpected packet type [%s]', jsonobj['discovertype']
104 
105  def processpkt_netconfig(self, drone, _unused_srcaddr, jsonobj):
106  '''We want to trigger ARP discovery when we hear a 'netconfig' packet
107 
108  Build up the parameters for the discovery
109  action, then send it to drone.request_discovery(...)
110  To build up the parameters, you use ConfigFile.agent_params()
111  which will pull values from the system configuration.
112  '''
113 
114  init_params = ConfigFile.agent_params(self.config, 'discovery', '#ARP', drone.designation)
115 
116  data = jsonobj['data'] # the data portion of the JSON message
117  discovery_args = []
118  for devname in data.keys():
119  #print >> sys.stderr, "*** devname:", devname
120  devinfo = data[devname]
121  if discovery_indicates_link_is_up(devinfo):
122  params = pyConfigContext(init_params)
123  params[CONFIGNAME_INSTANCE] = '_ARP_' + devname
124  params[CONFIGNAME_DEVNAME] = devname
125  #print >> sys.stderr, '#ARP parameters:', params
126  discovery_args.append(params)
127  if discovery_args:
128  drone.request_discovery(discovery_args)
129 
130  def processpkt_arp(self, drone, _unused_srcaddr, jsonobj):
131  '''We want to update the database when we hear a 'ARP' discovery packet
132  These discovery entries are the result of listening to ARP packets
133  in the nanoprobes. Some may already be in our database, and some may not be.
134 
135  As we process the packets we create any IPaddrNode and NICNode objects
136  that correspond to the things we've discovered. Since IP addresses
137  can move around, we potentially need to clean up relationships to
138  NICNodes - so that any given IP address is only associated with a single
139  MAC address.
140 
141  As noted in the class docs, the data we get is organized by IP address.
142  This means that a single MAC address (NIC) may appear multiple times in the
143  discovery data.
144 
145  One interesting question is what we should default the domain to for MACs and IPs
146  that we create. My thinking is that defaulting them to the domain of the Drone
147  that did the discovery is a reasonable choice.
148  '''
149 
150  data = jsonobj['data']
151  maciptable = {}
152  # Group the IP addresses by MAC address - reversing the map
153  for ip in data.keys():
154  mac = str(data[ip])
155  if mac not in maciptable:
156  maciptable[mac] = []
157  maciptable[mac].append(ip)
158 
159  for mac in maciptable:
160  self.filtered_add_mac_ip(drone, mac, maciptable[mac])
161 
162  def filtered_add_mac_ip(self, drone, macaddr, IPlist):
163  '''We process all the IP addresses that go with a given MAC address (NICNode)
164  The parameters are expected to be canonical address strings like str(pyNetAddr(...)).
165 
166  Lots of the information we're given is typically repeats of information we
167  were given before. This is why we keep these two in-memory maps
168  - to help speed that up by a huge factor.
169  '''
170  for ip in IPlist:
171  if ArpDiscoveryListener.ip_map.get(ip) != macaddr:
172  self.add_mac_ip(drone, macaddr, IPlist)
173  for ip in IPlist:
174  ArpDiscoveryListener.ip_map[ip] = macaddr
175  if macaddr not in ArpDiscoveryListener.mac_map:
176  ArpDiscoveryListener.mac_map[macaddr] = []
177  if ip not in ArpDiscoveryListener.mac_map[macaddr]:
178  ArpDiscoveryListener.mac_map[macaddr].append(ip)
179  return
180 
181  def add_mac_ip(self, drone, macaddr, IPlist):
182  '''We process all the IP addresses that go with a given MAC address (NICNode)
183  The parameters are expected to be canonical address strings like str(pyNetAddr(...)).
184  '''
185  nicnode = self.store.load_or_create(NICNode, domain=drone.domain, macaddr=macaddr)
186  if not Store.is_abstract(nicnode):
187  # This NIC already existed - let's see what IPs it already owned
188  currips = {}
189  oldiplist = self.store.load_related(nicnode, CMAconsts.REL_ipowner, IPaddrNode)
190  for ipnode in oldiplist:
191  currips[ipnode.ipaddr] = ipnode
192  #print >> sys.stderr, ('IP %s already related to NIC %s'
193  #% (str(ipnode.ipaddr), str(nicnode.macaddr)))
194  # See what IPs still need to be added
195  ips_to_add = []
196  for ip in IPlist:
197  if ip not in currips:
198  ips_to_add.append(ip)
199  # Replace the original list of IPs with those not already there...
200  IPlist = ips_to_add
201 
202  # Now we have a NIC and IPs which aren't already related to it
203  for ip in IPlist:
204  ipnode = self.store.load_or_create(IPaddrNode, domain=drone.domain, ipaddr=ip)
205  #print >> sys.stderr, ('CREATING IP %s for NIC %s'
206  #% (str(ipnode.ipaddr), str(nicnode.macaddr)))
207  if not Store.is_abstract(ipnode):
208  # Then this IP address already existed,
209  # but it wasn't related to our NIC...
210  # Make sure it isn't related to a different NIC
211  for oldnicnode in self.store.load_in_related(ipnode, CMAconsts.REL_ipowner,
212  GraphNode.factory):
213  self.store.separate(oldnicnode, CMAconsts.REL_ipowner, ipnode)
214  self.store.relate(nicnode, CMAconsts.REL_ipowner, ipnode)
def processpkt_arp(self, drone, _unused_srcaddr, jsonobj)
def add_mac_ip(self, drone, macaddr, IPlist)
def filtered_add_mac_ip(self, drone, macaddr, IPlist)
def processpkt(self, drone, srcaddr, jsonobj, discoverychanged)
Definition: arpdiscovery.py:93
def discovery_indicates_link_is_up(devinfo)
def processpkt_netconfig(self, drone, _unused_srcaddr, jsonobj)