The Assimilation Project  based on Assimilation version 1.1.6.1462601511
compressframe.c
Go to the documentation of this file.
1 
27 
33 
35 #define KBYTES(n) ((n)*1024)
36 
37 // Some of our data is very compressible - almost as bad as XML
38 // Hopefully we won't get _too_ many packets like this...
39 #define MAXUNCOMPRESSEDSIZE KBYTES(4096)
40 
41 #include <glib.h>
42 #include <fcntl.h>
43 #include <memory.h>
44 #include <malloc.h>
45 #include <compressframe.h>
46 #include <frameset.h>
47 #include <generic_tlv_min.h>
48 #include <tlvhelper.h>
49 
50 #ifdef HAVE_ZLIB_H
51 # define ZLIB_CONST 1 /* Enable const definitions where appropriate */
52 # include <zlib.h>
53 #endif /* HAVE_ZLIB_H */
54 #if ZLIB_VER_MAJOR > 1 || ZLIB_VER_MINOR > 2 || ZLIB_VER_REVISION > 7
55  // At least version 1.2.8 has the ZLIB_CONST #define to fix this
56 # define ZLIB_CONSTANT(e) ((const guint8*)(e))
57 #else
58  // Older zlib header files didn't declare things const like they should
59  // So we use a sleazy trick to deal with the broken header file :-(
60 # define ZLIB_CONSTANT(e) ((guint8*)(GSIZE_TO_POINTER(GPOINTER_TO_SIZE(e))))
61 #endif
62 
64 
66 FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify);
67 FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet*);
68 FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer);
69 FSTATIC int _compressframe_findmethod(int method);
70 FSTATIC gchar * _compressframe_toString(gconstpointer);
71 FSTATIC void assim_dump_bytes(char * prefix, gconstpointer p, int len);
72 
73 #define COMPFRAMESIZE 4 // one-byte compression type + 3 byte uncompressed length
74 
75 #ifdef HAVE_ZLIB_H
76 FSTATIC gpointer z_compressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
77 FSTATIC gpointer z_decompressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
78 #endif /* HAVE_ZLIB_H */
79 
81 static struct compression_types {
82  guint8 compression_type;
83  gpointer (*compress)(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
85  gpointer (*decompress)(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
87  const char* name;
88 }allcompressions [] = {
89 #ifdef HAVE_ZLIB_H
90  {COMPRESS_ZLIB, z_compressbuf, z_decompressbuf, "zlib"},
91 #endif
92 };
93 
94 #include <stdio.h>
95 FSTATIC void
96 assim_dump_bytes(char * prefix, gconstpointer p, int len)
97 {
98  int j;
99  const char * space = "";
100  fprintf(stderr, "%s", prefix);
101 
102  for (j=0; j < len; ++j) {
103  fprintf(stderr, "%s%02x", space, *(((const guint8*)p)+j));
104  space=" ";
105  }
106  fprintf(stderr, " [%d bytes]\n", len);
107 }
108 FSTATIC int
110 {
111  gsize j;
112  for (j=0; j < DIMOF(allcompressions); ++j) {
113  if (method == allcompressions[j].compression_type) {
114  return j;
115  }
116  }
117  return -1;
118 }
119 
120 //static void(*parentfinalize)(AssimObj*) = NULL;
122 compressframe_new(guint16 frame_type, guint16 compression_method)
123 {
124  Frame* fself;
125  CompressFrame* self;
126  int cmpindex;
127 
129 
130  if ((cmpindex = _compressframe_findmethod(compression_method)) < 0) {
131  g_warning("%s.%d: Unknown compression type: %d"
132  , __FUNCTION__, __LINE__, compression_method);
133  return(NULL);
134  }
135  fself = frame_new(frame_type, sizeof(CompressFrame));
136  self = NEWSUBCLASS(CompressFrame, fself);
137  self->compression_method = compression_method;
138  self->compression_index = cmpindex;
139  self->compression_threshold = DEFAULT_COMPRESSION_THRESHOLD;
140  fself->length = COMPFRAMESIZE;
145 #if 0
146  if (NULL == parentfinalize) {
147  parentfinalize = fself->baseclass._finalize;
148  }
150 #endif
151  return self;
152 }
153 
155 compressframe_new_string(guint16 frame_type, const char* compression_name)
156 {
157  gsize j;
158  for (j=0; j < DIMOF(allcompressions); ++j) {
159  if (strcmp(compression_name, allcompressions[j].name) == 0) {
160  return compressframe_new(frame_type, allcompressions[j].compression_type);
161  }
162  }
163  g_return_val_if_reached(NULL);
164 }
166 FSTATIC gboolean
167 _compressframe_isvalid(const Frame *fself, gconstpointer tlvstart, gconstpointer pktend)
168 {
169  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, fself);
170  guint8 compresstype;
171  guint32 origlen;
172  const guint8* valptr;
173  if (NULL == tlvstart) {
174  return ((gsize)self->compression_index) < DIMOF(allcompressions)
175  && allcompressions[self->compression_index].compression_type == self->compression_method;
176  }
177 
178  if ( ((const guint8*)pktend-(const guint8*)tlvstart) < 12
179  || get_generic_tlv_len(tlvstart, pktend) <= 8) {
180  return FALSE;
181  }
182  valptr = get_generic_tlv_value(tlvstart, pktend);
183  compresstype = tlv_get_guint8(valptr, pktend);
184  g_return_val_if_fail(_compressframe_findmethod(compresstype) >= 0, FALSE);
185  origlen = tlv_get_guint24(valptr+1, pktend);
186  // Trying to avoid a DOS attack using huge packets
187  // But typically they are signed - so our attacker would have to have control over
188  // a nanoprobe on one of our machines.
189  if (origlen > MAXUNCOMPRESSEDSIZE) {
190  g_warning("%s.%d: Excessively large compressed packet: %ld bytes (%ld compressed)"
191  , __FUNCTION__, __LINE__
192  , (long)origlen, (long)get_generic_tlv_len(tlvstart, pktend));
193  return FALSE;
194  }
195  return TRUE;
196 }
197 
198 FSTATIC void
199 _compressframe_setvalue(Frame *f, gpointer value, guint32 len, GDestroyNotify valnotify)
200 {
201  (void)f;
202  (void)value;
203  (void)len;
204  (void)valnotify;
205  g_warning("%s:%d: Not possible to set the value of a CompressFrame", __FUNCTION__, __LINE__);
206 }
207 
208 FSTATIC gchar *
209 _compressframe_toString(gconstpointer aself)
210 {
211  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, aself);
212  double ratio;
213  if (self->baseclass.length <= 4) {
214  return g_strdup_printf("CompressFrame(frametype:%d, method:%s)"
215  , self->baseclass.type, allcompressions[self->compression_index].name);
216  }
217  ratio = (double)self->decompressed_size / ((double)(self->baseclass.length-4));
218  return g_strdup_printf("CompressFrame(frametype:%d, method:%s, len:%d uncompsize:%d, ratio:%0.2f:1)"
219  , self->baseclass.type, allcompressions[self->compression_index].name
220  , self->baseclass.length, self->decompressed_size, ratio);
221 }
222 
223 
229 FSTATIC void
230 _compressframe_updatedata(Frame *f, //<[in] Our frame ("self")
231  gpointer tlvstart, //<[in] Beginning of where to put our data (not really)
232  gconstpointer pktend, //<[out] Where to put the final data (not really)
233  FrameSet* fs) //<[in/out] Frameset to update the "packet" for
234  //< once we compress this data
235 {
237  guint8* pktstart = fs->packet;
238  guint8* tlvstart8 = tlvstart;
239  const guint8* pktend8 = pktend;
240  guint8* valptr;
241  guint32 ouroffset; // Offset to beginning of our packet
242  guint32 cmpoffset; // Offset to beginning of compressed data
243  guint8* newpacket;
244  guint8* newpktend;
245  int compressedsize;
246 
247 
248  // Now on to our side effect - compressing the frames that follow us...
249  ouroffset = (tlvstart8-pktstart);
250  cmpoffset = ouroffset + COMPFRAMESIZE+FRAME_INITSIZE;
251  newpacket = allcompressions[self->compression_index].compress
252  (pktstart, pktend8-pktstart, cmpoffset, MAXUDPSIZE, &compressedsize, 0);
253  self->decompressed_size = (pktend8 - pktstart) - cmpoffset;
254 
255  if (NULL == newpacket) {
256  g_warning("%s:%d: Unable to compress %d byte packet to %d byte UDP packet"
257  , __FUNCTION__, __LINE__, self->decompressed_size, MAXUDPSIZE);
258  }
259  newpktend = newpacket + compressedsize;
260  // Write our type and length into the (new) packet
261  set_generic_tlv_type(newpacket+ouroffset, f->type, newpktend);
262  self->baseclass.length = (compressedsize-cmpoffset) + COMPFRAMESIZE;
263  set_generic_tlv_len(newpacket+ouroffset, self->baseclass.length, newpktend);
264  valptr = get_generic_tlv_nonconst_value(newpacket+ouroffset, newpktend);
265  // Our TLV value consists of the compression method followed by a 3 byte
266  // packet length, followed by the compressed data (already in "newpacket").
267  // This restricts us to a 16M decompressed original packet.
268  // Since this has to compress down to a single UDP packet,
269  // this a very reasonable assumption...
270  // In practice, our JSON seems to be limited to about 300K decompressed.
271  tlv_set_guint8(valptr, self->compression_method, newpktend);
272  tlv_set_guint24(valptr+1, self->decompressed_size, newpktend);
273  if (fs->packet) {
274  FREE(fs->packet);
275  }
276  fs->packet = newpacket;
277  fs->pktend = newpktend;
278 }
279 
280 #define COMPRESSFRAMEMIN 4
281 FSTATIC Frame*
282 compressframe_tlvconstructor(gpointer tlvstart,
283  gconstpointer pktend,
284  gpointer* newpacket,
285  gpointer* newpacketend)
286 {
287  const guint8* pktend8 = pktend;
288  const guint8* tlvstart8 = tlvstart;
289  const guint8* packet;
290  const guint8* valueptr;
291  guint32 cmppktsize;
292  guint8 compression_type;
293  guint32 decompressed_size;
294  int actual_size;
295  int compression_index;
296  guint16 frametype;
298  CompressFrame* ret;
299  /* Our four bytes of real data are:
300  * 1-byte compression type
301  * 3-byte decompressed size
302  */
303  frametype = get_generic_tlv_type(tlvstart, pktend);
304  valueptr = get_generic_tlv_value(tlvstart, pktend);
305  compression_type = tlv_get_guint8(valueptr, pktend);
306  decompressed_size = tlv_get_guint24(valueptr+1, pktend);
307  compression_index = _compressframe_findmethod(compression_type);
308  // Trying to mitigate possible DOS attack using huge packets
309  if (decompressed_size > MAXUNCOMPRESSEDSIZE) {
310  g_warning("%s.%d: Excessively large compressed packet: %ld bytes (%ld compressed)"
311  , __FUNCTION__, __LINE__
312  , (long)decompressed_size, (long)get_generic_tlv_len(tlvstart, pktend));
313  return FALSE;
314  }
315  g_return_val_if_fail(decompressed_size > 16, NULL);
316  g_return_val_if_fail(compression_index >= 0, NULL);
317  packet = valueptr + COMPRESSFRAMEMIN;
318 
319  cmppktsize = pktend8 - tlvstart8; // Compressed packet size
320 
321  *newpacket = allcompressions[compression_index].decompress
322  (packet, cmppktsize, 0, decompressed_size, &actual_size);
323  g_return_val_if_fail(*newpacket != NULL, NULL);
324  *newpacketend = (guint8*)*newpacket + actual_size;
325 
326  ret = compressframe_new(frametype, compression_type);
327  g_return_val_if_fail(ret != NULL, NULL);
328  ret->decompressed_size = decompressed_size;
329  return &ret->baseclass;
330 }
331 
332 #ifdef HAVE_ZLIB_H
333 gpointer
339 z_compressbuf(gconstpointer inbuf
340 , int insize
341 , int offset
342 , int maxout
343 , int *actualsize
344 , int level)
345 {
346 
347  guint8* outbuf;
348  z_stream stream;
349  int ret;
350 #if 0
351  int space;
352  double ratio;
353 #endif
354 
355  // Compute compression level
356  // If our guess doesn't work, we'll escalate to max compression
357  // This adds compression expense, this is mostly in the nanoprobe side, so we don't much care.
358  if (level < 1) {
359  if (insize < KBYTES(189)) {
360  level = 1;
361  }else if (insize < KBYTES(225)) {
362  level = 6;
363  }else{
364  level = 9;
365  }
366  }
367 
368  /* Set up libz */
369  stream.zalloc = Z_NULL;
370  stream.zfree = Z_NULL;
371  stream.opaque = Z_NULL;
372  ret = deflateInit(&stream, level);
373 
374  if (ret != Z_OK) {
375  g_warning("%s.%d: OOPS, got a deflateInit return of %d"
376  , __FUNCTION__, __LINE__, ret);
377  return NULL;
378  }
379 #if 0
380  outbuf = g_malloc(maxout);
381 #else
382  outbuf = calloc(maxout, 1);
383 #endif
384  if (NULL == outbuf) {
385  g_warning("%s.%d: OOPS out of space.", __FUNCTION__, __LINE__);
386  return NULL;
387  }
388  stream.avail_in = insize-offset;
389  stream.next_in = ZLIB_CONSTANT(inbuf)+offset;
390  stream.avail_out = maxout-offset;
391  stream.next_out = outbuf+offset;
392 
393  /* Compress it */
394  ret = deflate(&stream, Z_FINISH);
395  if (ret != Z_STREAM_END) {
396  g_free(outbuf);
397  if (level < 9) {
398  return z_compressbuf(inbuf, insize, offset, maxout, actualsize, 9);
399  }
400  g_warning("%s.%d: Got return code %d from deflate."
401  , __FUNCTION__, __LINE__, ret);
402  return NULL;
403  }
404 #if 0
405  {
406  double ratio;
407  ratio = ((double)stream.total_in/((double)stream.total_out));
408  fprintf(stderr, "Compressing %ld bytes into %ld bytes with level %d ratio %.2f:1\n"
409  , stream.total_in, stream.total_out, level, ratio);
410  }
411 #endif
412  (void)deflateEnd(&stream);
413  *actualsize = stream.total_out + offset;
414  outbuf = g_realloc(outbuf, *actualsize);
415  return outbuf;
416 }
417 
420 gpointer
421 z_decompressbuf(gconstpointer inbuf
422 , int insize
423 , int offset
424 , int maxout
426 , int *uncompsize)
428 {
429  gpointer outbuf;
430  int outsize;
431  int ret;
432  z_stream stream;
433  /* Set up libz */
434  stream.zalloc = Z_NULL;
435  stream.zfree = Z_NULL;
436  stream.opaque = Z_NULL;
437  stream.avail_in = insize-offset;
438  stream.next_in = ZLIB_CONSTANT(inbuf) + offset;
439  g_return_val_if_fail (Z_OK == inflateInit(&stream), NULL);
440  outbuf = g_malloc(maxout);
441  if (offset > 0) {
442  memcpy(outbuf, inbuf, offset);
443  }
444  stream.avail_out = maxout;
445  stream.next_out = ((guint8 *)outbuf) + offset;
446  // Decompress our input buffer.
447  ret = inflate(&stream, Z_FINISH);
448  (void)inflateEnd(&stream);
449  if (ret != Z_STREAM_END) {
450  g_warning( "%s.%d: GOT inflate RETURN OF %d"
451  , __FUNCTION__, __LINE__, ret);
452  g_free(outbuf);
453  return NULL;
454  }
455  outsize = maxout - stream.avail_out;
456  if (outsize > maxout) {
457  outbuf = g_realloc(outbuf, outsize);
458  }
459  (void)inflateEnd(&stream);
460  *uncompsize = outsize;
461  return outbuf;
462 }
463 #endif /* HAVE_ZLIB_H */
#define FRAME_INITSIZE
Definition: frame.h:59
guint32 length
Frame Length.
Definition: frame.h:46
FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify)
guint8 tlv_get_guint8(const void *vitem, const void *bufend)
Retrieve an unsigned 8 bit integer from the given location with error checking.
Definition: tlvhelper.c:31
AssimObj baseclass
Base object class for our Class system.
Definition: frame.h:44
Implements minimal client-oriented Frame and Frameset capabilities.
This is the base Frame class object (in-memory TLV (type, length, value)) for every general component...
Definition: frame.h:43
#define ZLIB_CONSTANT(e)
Definition: compressframe.c:60
gpointer pktend
Last byte past the end of the packet.
Definition: frameset.h:51
#define FSTATIC
Definition: projectcommon.h:31
#define KBYTES(n)
Definition: compressframe.c:35
void(* setvalue)(Frame *self, gpointer value, guint32 length, GDestroyNotify valfinal)
member function for setting value
Definition: frame.h:52
FSTATIC void _compressframe_finalize(AssimObj *self)
void tlv_set_guint24(void *vitem, guint32 item, const void *bufend)
Set an unsigned 24 bit (3-byte) integer to the given location with error checking and without caring ...
Definition: tlvhelper.c:148
guint16 get_generic_tlv_type(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Type' of the given TLV TLV entry (first two bytes)
#define BINDDEBUG(Cclass)
BINDDEBUG is for telling the class system where the debug variable for this class is - put it in the ...
Definition: proj_classes.h:82
#define __FUNCTION__
CompressFrame * compressframe_new(guint16 frame_type, guint16 compression_method)
This is our CompressFrame class object - used for representing a compression method.
Definition: compressframe.h:41
#define MAXUNCOMPRESSEDSIZE
Definition: compressframe.c:39
Frame * frame_new(guint16 frame_type, gsize framesize)
Construct a new frame - allowing for "derived" frame types...
Definition: frame.c:125
#define COMPRESS_ZLIB
Compression using 'zlib'.
Definition: compressframe.h:38
Describes interfaces to C-String Frame (Compressframe) C-Class It holds conventional zero-terminated ...
#define COMPFRAMESIZE
Definition: compressframe.c:73
#define FREE(m)
Our interface to free.
Definition: projectcommon.h:29
gpointer packet
Pointer to packet (when constructed)
Definition: frameset.h:50
guint32 tlv_get_guint24(const void *vitem, const void *bufend)
Retrieve an unsigned 24 bit (3-byte) integer from the given location with error checking and without ...
Definition: tlvhelper.c:133
gchar *(* toString)(gconstpointer)
Produce malloc-ed string representation.
Definition: assimobj.h:58
#define CASTTOCONSTCLASS(Cclass, obj)
Safely cast 'obj' to const C-class 'class' - verifying that it was registered as being of type class ...
Definition: proj_classes.h:71
void set_generic_tlv_type(gpointer tlv_vp, guint16 newtype, gconstpointer pktend)
Set the 'Type' of the given TLV TLV entry (first two bytes)
guint32 get_generic_tlv_len(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Length' of the given generic TLV entry (first 3 bytes after type) Note: return result is ...
#define DIMOF(a)
Definition: lldp_min.c:30
TLV helper interfaces definitions.
void tlv_set_guint8(void *vitem, guint8 item, const void *bufend)
Set an unsigned 8 bit integer to the given location with error checking.
Definition: tlvhelper.c:42
gboolean(* isvalid)(const Frame *self, gconstpointer tlvptr, gconstpointer pktend)
TRUE if TLV data looks valid...
Definition: frame.h:50
#define DEFAULT_COMPRESSION_THRESHOLD
Default value of when to start compressing.
Definition: compressframe.h:34
guint32 decompressed_size
Definition: compressframe.h:44
#define COMPRESSFRAMEMIN
void set_generic_tlv_len(gpointer tlv_vp, guint32 newsize, gconstpointer pktend)
Set the 'Length' of the given generic TLV entry (first 3 bytes after type)
guint16 type
Frame Type (see Individual TLV 'Frame' data types and layouts (by TLV type) - frameformats.h )
Definition: frame.h:45
Provides definitions for using our generic TLV capabilities.
FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer)
Return TRUE if this is a valid CompressFrame - either an object or on-the-wire version.
gconstpointer get_generic_tlv_value(gconstpointer tlv_vp, gconstpointer pktend)
Return a const pointer to the 'Value' of the given generic TLV entry.
DEBUGDECLARATIONS
Definition: compressframe.c:63
FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet *)
We update the data in the packet from our CompressFrame object AND ALSO have the side-effect of compr...
FSTATIC void assim_dump_bytes(char *prefix, gconstpointer p, int len)
Definition: compressframe.c:96
void(* updatedata)(Frame *self, gpointer tlvptr, gconstpointer pktend, FrameSet *fs)
Update packet data.
Definition: frame.h:49
FSTATIC gchar * _compressframe_toString(gconstpointer)
FrameSet class - used for collecting Frames when not on the wire, and for marshalling/demarshalling t...
Definition: frameset.h:46
FSTATIC int _compressframe_findmethod(int method)
gpointer get_generic_tlv_nonconst_value(gpointer tlv_vp, gconstpointer pktend)
Return a non-const (mutable) pointer to the 'Value' of the given generic TLV entry.
CompressFrame * compressframe_new_string(guint16 frame_type, const char *compression_name)
void(* _finalize)(AssimObj *)
Free object (private)
Definition: assimobj.h:55
FSTATIC Frame * compressframe_tlvconstructor(gpointer tlvstart, gconstpointer pktend, gpointer *newpacket, gpointer *newpacketend)
#define CASTTOCLASS(Cclass, obj)
Safely cast 'obj' to C-class 'class' - verifying that it was registerd as being of type class ...
Definition: proj_classes.h:66
#define NEWSUBCLASS(Cclass, obj)
Definition: proj_classes.h:67
#define MAXUDPSIZE
Maximum UDP packet size.
Definition: compressframe.h:33