The Assimilation Project
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
compressframe.c
Go to the documentation of this file.
1 
27 
33 
35 #define KBYTES(n) ((n)*1024)
36 #define MAXUNCOMPRESSEDSIZE KBYTES(1024)
39 
40 #include <glib.h>
41 #include <fcntl.h>
42 #include <memory.h>
43 #include <malloc.h>
44 #include <compressframe.h>
45 #include <frameset.h>
46 #include <generic_tlv_min.h>
47 #include <tlvhelper.h>
48 
49 #ifdef HAVE_ZLIB_H
50 # define ZLIB_CONST 1 /* Enable const definitions where appropriate */
51 # include <zlib.h>
52 #endif /* HAVE_ZLIB_H */
53 #if ZLIB_VER_MAJOR > 1 || ZLIB_VER_MINOR > 2 || ZLIB_VER_REVISION > 7
54  // At least version 1.2.8 has the ZLIB_CONST #define to fix this
55 # define ZLIB_CONSTANT(e) ((const guint8*)(e))
56 #else
57  // Older zlib header files didn't declare things const like they should
58  // So we use a sleazy trick to deal with the broken header file :-(
59 # define ZLIB_CONSTANT(e) ((guint8*)(GSIZE_TO_POINTER(GPOINTER_TO_SIZE(e))))
60 #endif
61 
63 
65 FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify);
66 FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet*);
67 FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer);
68 FSTATIC int _compressframe_findmethod(int method);
69 FSTATIC gchar * _compressframe_toString(gconstpointer);
70 FSTATIC void assim_dump_bytes(char * prefix, gconstpointer p, int len);
71 
72 #define COMPFRAMESIZE 4 // one-byte compression type + 3 byte uncompressed length
73 
74 #ifdef HAVE_ZLIB_H
75 FSTATIC gpointer z_compressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
76 FSTATIC gpointer z_decompressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
77 #endif /* HAVE_ZLIB_H */
78 
80 static struct compression_types {
81  guint8 compression_type;
82  gpointer (*compress)(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
84  gpointer (*decompress)(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
86  const char* name;
87 }allcompressions [] = {
88 #ifdef HAVE_ZLIB_H
89  {COMPRESS_ZLIB, z_compressbuf, z_decompressbuf, "zlib"},
90 #endif
91 };
92 
93 #include <stdio.h>
94 FSTATIC void
95 assim_dump_bytes(char * prefix, gconstpointer p, int len)
96 {
97  int j;
98  const char * space = "";
99  fprintf(stderr, "%s", prefix);
100 
101  for (j=0; j < len; ++j) {
102  fprintf(stderr, "%s%02x", space, *(((const guint8*)p)+j));
103  space=" ";
104  }
105  fprintf(stderr, " [%d bytes]\n", len);
106 }
107 FSTATIC int
109 {
110  gsize j;
111  for (j=0; j < DIMOF(allcompressions); ++j) {
112  if (method == allcompressions[j].compression_type) {
113  return j;
114  }
115  }
116  return -1;
117 }
118 
119 //static void(*parentfinalize)(AssimObj*) = NULL;
121 compressframe_new(guint16 frame_type, guint16 compression_method)
122 {
123  Frame* fself;
124  CompressFrame* self;
125  int cmpindex;
126 
128 
129  if ((cmpindex = _compressframe_findmethod(compression_method)) < 0) {
130  g_warning("%s.%d: Unknown compression type: %d"
131  , __FUNCTION__, __LINE__, compression_method);
132  return(NULL);
133  }
134  fself = frame_new(frame_type, sizeof(CompressFrame));
135  self = NEWSUBCLASS(CompressFrame, fself);
136  self->compression_method = compression_method;
137  self->compression_index = cmpindex;
138  self->compression_threshold = DEFAULT_COMPRESSION_THRESHOLD;
139  fself->length = COMPFRAMESIZE;
144 #if 0
145  if (NULL == parentfinalize) {
146  parentfinalize = fself->baseclass._finalize;
147  }
149 #endif
150  return self;
151 }
152 
154 compressframe_new_string(guint16 frame_type, const char* compression_name)
155 {
156  gsize j;
157  for (j=0; j < DIMOF(allcompressions); ++j) {
158  if (strcmp(compression_name, allcompressions[j].name) == 0) {
159  return compressframe_new(frame_type, allcompressions[j].compression_type);
160  }
161  }
162  g_return_val_if_reached(NULL);
163 }
165 FSTATIC gboolean
166 _compressframe_isvalid(const Frame *fself, gconstpointer tlvstart, gconstpointer pktend)
167 {
168  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, fself);
169  guint8 compresstype;
170  guint32 origlen;
171  const guint8* valptr;
172  if (NULL == tlvstart) {
173  return ((gsize)self->compression_index) < DIMOF(allcompressions)
174  && allcompressions[self->compression_index].compression_type == self->compression_method;
175  }
176 
177  if ( ((const guint8*)pktend-(const guint8*)tlvstart) < 12
178  || get_generic_tlv_len(tlvstart, pktend) <= 8) {
179  return FALSE;
180  }
181  valptr = get_generic_tlv_value(tlvstart, pktend);
182  compresstype = tlv_get_guint8(valptr, pktend);
183  g_return_val_if_fail(_compressframe_findmethod(compresstype) >= 0, FALSE);
184  origlen = tlv_get_guint24(valptr+1, pktend);
185  // Trying to avoid a DOS attack using huge packets
186  g_return_val_if_fail(origlen <= MAXUNCOMPRESSEDSIZE || origlen >= 16, FALSE);
187  return TRUE;
188 }
189 
190 FSTATIC void
191 _compressframe_setvalue(Frame *f, gpointer value, guint32 len, GDestroyNotify valnotify)
192 {
193  (void)f;
194  (void)value;
195  (void)len;
196  (void)valnotify;
197  g_warning("%s:%d: Not possible to set the value of a CompressFrame", __FUNCTION__, __LINE__);
198 }
199 
200 FSTATIC gchar *
201 _compressframe_toString(gconstpointer aself)
202 {
203  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, aself);
204  double ratio;
205  if (self->baseclass.length <= 4) {
206  return g_strdup_printf("CompressFrame(frametype:%d, method:%s)"
207  , self->baseclass.type, allcompressions[self->compression_index].name);
208  }
209  ratio = (double)self->decompressed_size / ((double)(self->baseclass.length-4));
210  return g_strdup_printf("CompressFrame(frametype:%d, method:%s, len:%d uncompsize:%d, ratio:%0.2f:1)"
211  , self->baseclass.type, allcompressions[self->compression_index].name
212  , self->baseclass.length, self->decompressed_size, ratio);
213 }
214 
215 
221 FSTATIC void
222 _compressframe_updatedata(Frame *f, //<[in] Our frame ("self")
223  gpointer tlvstart, //<[in] Beginning of where to put our data (not really)
224  gconstpointer pktend, //<[out] Where to put the final data (not really)
225  FrameSet* fs) //<[in/out] Frameset to update the "packet" for
226  //< once we compress this data
227 {
229  guint8* pktstart = fs->packet;
230  guint8* tlvstart8 = tlvstart;
231  const guint8* pktend8 = pktend;
232  guint8* valptr;
233  guint32 ouroffset; // Offset to beginning of our packet
234  guint32 cmpoffset; // Offset to beginning of compressed data
235  guint8* newpacket;
236  guint8* newpktend;
237  int compressedsize;
238 
239 
240  // Now on to our side effect - compressing the frames that follow us...
241  ouroffset = (tlvstart8-pktstart);
242  cmpoffset = ouroffset + COMPFRAMESIZE+FRAME_INITSIZE;
243  newpacket = allcompressions[self->compression_index].compress
244  (pktstart, pktend8-pktstart, cmpoffset, MAXUDPSIZE, &compressedsize, 0);
245  self->decompressed_size = (pktend8 - pktstart) - cmpoffset;
246 
247  if (NULL == newpacket) {
248  g_warning("%s:%d: Unable to compress %d byte packet to %d byte UDP packet"
249  , __FUNCTION__, __LINE__, self->decompressed_size, MAXUDPSIZE);
250  }
251  newpktend = newpacket + compressedsize;
252  // Write our type and length into the (new) packet
253  set_generic_tlv_type(newpacket+ouroffset, f->type, newpktend);
254  self->baseclass.length = (compressedsize-cmpoffset) + COMPFRAMESIZE;
255  set_generic_tlv_len(newpacket+ouroffset, self->baseclass.length, newpktend);
256  valptr = get_generic_tlv_nonconst_value(newpacket+ouroffset, newpktend);
257  // Our TLV value consists of the compression method followed by a 3 byte
258  // packet length, followed by the compressed data (already in "newpacket").
259  // This restricts us to a 16M decompressed original packet.
260  // Since this has to compress down to a single UDP packet,
261  // this a very reasonable assumption...
262  // In practice, our JSON seems to be limited to about 300K decompressed.
263  tlv_set_guint8(valptr, self->compression_method, newpktend);
264  tlv_set_guint24(valptr+1, self->decompressed_size, newpktend);
265  fs->packet = newpacket;
266  fs->pktend = newpktend;
267 }
268 
269 #define COMPRESSFRAMEMIN 4
270 FSTATIC Frame*
271 compressframe_tlvconstructor(gpointer tlvstart,
272  gconstpointer pktend,
273  gpointer* newpacket,
274  gpointer* newpacketend)
275 {
276  const guint8* pktend8 = pktend;
277  const guint8* tlvstart8 = tlvstart;
278  const guint8* packet;
279  const guint8* valueptr;
280  guint32 cmppktsize;
281  guint8 compression_type;
282  guint32 decompressed_size;
283  int actual_size;
284  int compression_index;
285  guint16 frametype;
287  CompressFrame* ret;
288  /* Our four bytes of real data are:
289  * 1-byte compression type
290  * 3-byte decompressed size
291  */
292  frametype = get_generic_tlv_type(tlvstart, pktend);
293  valueptr = get_generic_tlv_value(tlvstart, pktend);
294  compression_type = tlv_get_guint8(valueptr, pktend);
295  decompressed_size = tlv_get_guint24(valueptr+1, pktend);
296  compression_index = _compressframe_findmethod(compression_type);
297  // Trying to mitigate possible DOS attack using huge packets
298  // In practice, our max JSON decompressed size is under 325K
299  g_return_val_if_fail(decompressed_size <= MAXUNCOMPRESSEDSIZE, NULL);
300  g_return_val_if_fail(decompressed_size > 16, NULL);
301  g_return_val_if_fail(compression_index >= 0, NULL);
302  packet = valueptr + COMPRESSFRAMEMIN;
303 
304  cmppktsize = pktend8 - tlvstart8; // Compressed packet size
305 
306  *newpacket = allcompressions[compression_index].decompress
307  (packet, cmppktsize, 0, decompressed_size, &actual_size);
308  g_return_val_if_fail(*newpacket != NULL, NULL);
309  *newpacketend = (guint8*)*newpacket + actual_size;
310 
311  ret = compressframe_new(frametype, compression_type);
312  g_return_val_if_fail(ret != NULL, NULL);
313  ret->decompressed_size = decompressed_size;
314  return &ret->baseclass;
315 }
316 
317 #ifdef HAVE_ZLIB_H
318 gpointer
324 z_compressbuf(gconstpointer inbuf
325 , int insize
326 , int offset
327 , int maxout
328 , int *actualsize
329 , int level)
330 {
331 
332  guint8* outbuf;
333  z_stream stream;
334  int ret;
335 #if 0
336  int space;
337  double ratio;
338 #endif
339 
340  // Compute compression level
341  // If our guess doesn't work, we'll escalate to max compression
342  // This adds compression expense, this is mostly in the nanoprobe side, so we don't much care.
343  if (level < 1) {
344  if (insize < KBYTES(189)) {
345  level = 1;
346  }else if (insize < KBYTES(225)) {
347  level = 6;
348  }else{
349  level = 9;
350  }
351  }
352 
353  /* Set up libz */
354  stream.zalloc = Z_NULL;
355  stream.zfree = Z_NULL;
356  stream.opaque = Z_NULL;
357  ret = deflateInit(&stream, level);
358 
359  if (ret != Z_OK) {
360  g_warning("%s.%d: OOPS, got a deflateInit return of %d"
361  , __FUNCTION__, __LINE__, ret);
362  return NULL;
363  }
364 #if 0
365  outbuf = g_malloc(maxout);
366 #else
367  outbuf = calloc(maxout, 1);
368 #endif
369  if (NULL == outbuf) {
370  g_warning("%s.%d: OOPS out of space.", __FUNCTION__, __LINE__);
371  return NULL;
372  }
373  stream.avail_in = insize-offset;
374  stream.next_in = ZLIB_CONSTANT(inbuf)+offset;
375  stream.avail_out = maxout-offset;
376  stream.next_out = outbuf+offset;
377 
378  /* Compress it */
379  ret = deflate(&stream, Z_FINISH);
380  if (ret != Z_STREAM_END) {
381  g_free(outbuf);
382  if (level < 9) {
383  return z_compressbuf(inbuf, insize, offset, maxout, actualsize, 9);
384  }
385  g_warning("%s.%d: Got return code %d from deflate."
386  , __FUNCTION__, __LINE__, ret);
387  return NULL;
388  }
389 #if 0
390  {
391  double ratio;
392  ratio = ((double)stream.total_in/((double)stream.total_out));
393  fprintf(stderr, "Compressing %ld bytes into %ld bytes with level %d ratio %.2f:1\n"
394  , stream.total_in, stream.total_out, level, ratio);
395  }
396 #endif
397  (void)deflateEnd(&stream);
398  *actualsize = stream.total_out + offset;
399  outbuf = g_realloc(outbuf, *actualsize);
400  return outbuf;
401 }
402 
405 gpointer
406 z_decompressbuf(gconstpointer inbuf
407 , int insize
408 , int offset
409 , int maxout
411 , int *uncompsize)
413 {
414  gpointer outbuf;
415  int outsize;
416  int ret;
417  z_stream stream;
418  /* Set up libz */
419  stream.zalloc = Z_NULL;
420  stream.zfree = Z_NULL;
421  stream.opaque = Z_NULL;
422  stream.avail_in = insize-offset;
423  stream.next_in = ZLIB_CONSTANT(inbuf) + offset;
424  g_return_val_if_fail (Z_OK == inflateInit(&stream), NULL);
425  outbuf = g_malloc(maxout);
426  if (offset > 0) {
427  memcpy(outbuf, inbuf, offset);
428  }
429  stream.avail_out = maxout;
430  stream.next_out = ((guint8 *)outbuf) + offset;
431  // Decompress our input buffer.
432  ret = inflate(&stream, Z_FINISH);
433  (void)inflateEnd(&stream);
434  if (ret != Z_STREAM_END) {
435  g_warning( "%s.%d: GOT inflate RETURN OF %d"
436  , __FUNCTION__, __LINE__, ret);
437  g_free(outbuf);
438  return NULL;
439  }
440  outsize = maxout - stream.avail_out;
441  if (outsize > maxout) {
442  outbuf = g_realloc(outbuf, outsize);
443  }
444  (void)inflateEnd(&stream);
445  *uncompsize = outsize;
446  return outbuf;
447 }
448 #endif /* HAVE_ZLIB_H */
#define FRAME_INITSIZE
Definition: frame.h:59
guint32 length
Frame Length.
Definition: frame.h:46
FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify)
guint8 tlv_get_guint8(const void *vitem, const void *bufend)
Retrieve an unsigned 8 bit integer from the given location with error checking.
Definition: tlvhelper.c:30
AssimObj baseclass
Base object class for our Class system.
Definition: frame.h:44
Implements minimal client-oriented Frame and Frameset capabilities.
This is the base Frame class object (in-memory TLV (type, length, value)) for every general component...
Definition: frame.h:43
#define ZLIB_CONSTANT(e)
Definition: compressframe.c:59
gpointer pktend
Last byte past the end of the packet.
Definition: frameset.h:51
#define FSTATIC
Definition: projectcommon.h:31
#define KBYTES(n)
Definition: compressframe.c:35
FSTATIC void _compressframe_finalize(AssimObj *self)
void tlv_set_guint24(void *vitem, guint32 item, const void *bufend)
Set an unsigned 24 bit (3-byte) integer to the given location with error checking and without caring ...
Definition: tlvhelper.c:146
guint16 get_generic_tlv_type(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Type' of the given TLV TLV entry (first two bytes)
#define BINDDEBUG(Cclass)
BINDDEBUG is for telling the class system where the debug variable for this class is - put it in the ...
Definition: proj_classes.h:82
#define __FUNCTION__
CompressFrame * compressframe_new(guint16 frame_type, guint16 compression_method)
This is our CompressFrame class object - used for representing a compression method.
Definition: compressframe.h:41
#define MAXUNCOMPRESSEDSIZE
In practice, our max JSON decompressed size is under 325K, so 1M seems safe more or less not matter w...
Definition: compressframe.c:38
Frame * frame_new(guint16 frame_type, gsize framesize)
Construct a new frame - allowing for "derived" frame types...
Definition: frame.c:124
#define COMPRESS_ZLIB
Compression using 'zlib'.
Definition: compressframe.h:38
gboolean(* isvalid)(const Frame *self, gconstpointer tlvptr, gconstpointer pktend)
TRUE if TLV data looks valid...
Definition: frame.h:50
Describes interfaces to C-String Frame (Compressframe) C-Class It holds conventional zero-terminated ...
#define COMPFRAMESIZE
Definition: compressframe.c:72
gpointer packet
Pointer to packet (when constructed)
Definition: frameset.h:50
guint32 tlv_get_guint24(const void *vitem, const void *bufend)
Retrieve an unsigned 24 bit (3-byte) integer from the given location with error checking and without ...
Definition: tlvhelper.c:131
#define CASTTOCONSTCLASS(Cclass, obj)
Safely cast 'obj' to const C-class 'class' - verifying that it was registered as being of type class ...
Definition: proj_classes.h:71
void set_generic_tlv_type(gpointer tlv_vp, guint16 newtype, gconstpointer pktend)
Set the 'Type' of the given TLV TLV entry (first two bytes)
guint32 get_generic_tlv_len(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Length' of the given generic TLV entry (first 3 bytes after type)
#define DIMOF(a)
Definition: lldp_min.c:30
TLV helper interfaces definitions.
void tlv_set_guint8(void *vitem, guint8 item, const void *bufend)
Set an unsigned 8 bit integer to the given location with error checking.
Definition: tlvhelper.c:41
#define DEFAULT_COMPRESSION_THRESHOLD
Default value of when to start compressing.
Definition: compressframe.h:34
guint32 decompressed_size
Definition: compressframe.h:44
#define COMPRESSFRAMEMIN
void set_generic_tlv_len(gpointer tlv_vp, guint32 newsize, gconstpointer pktend)
Set the 'Length' of the given generic TLV entry (first 3 bytes after type)
void(* _finalize)(AssimObj *)
Free object (private)
Definition: assimobj.h:55
guint16 type
Frame Type (see Individual TLV 'Frame' data types and layouts (by TLV type) - frameformats.h )
Definition: frame.h:45
Provides definitions for using our generic TLV capabilities.
FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer)
Return TRUE if this is a valid CompressFrame - either an object or on-the-wire version.
gconstpointer get_generic_tlv_value(gconstpointer tlv_vp, gconstpointer pktend)
Return a const pointer to the 'Value' of the given generic TLV entry.
DEBUGDECLARATIONS
Definition: compressframe.c:62
FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet *)
We update the data in the packet from our CompressFrame object AND ALSO have the side-effect of compr...
FSTATIC void assim_dump_bytes(char *prefix, gconstpointer p, int len)
Definition: compressframe.c:95
FSTATIC gchar * _compressframe_toString(gconstpointer)
void(* updatedata)(Frame *self, gpointer tlvptr, gconstpointer pktend, FrameSet *fs)
Update packet data.
Definition: frame.h:49
FrameSet class - used for collecting Frames when not on the wire, and for marshalling/demarshalling t...
Definition: frameset.h:46
gchar *(* toString)(gconstpointer)
Produce malloc-ed string representation.
Definition: assimobj.h:58
FSTATIC int _compressframe_findmethod(int method)
gpointer get_generic_tlv_nonconst_value(gpointer tlv_vp, gconstpointer pktend)
Return a non-const (mutable) pointer to the 'Value' of the given generic TLV entry.
CompressFrame * compressframe_new_string(guint16 frame_type, const char *compression_name)
void(* setvalue)(Frame *self, gpointer value, guint32 length, GDestroyNotify valfinal)
member function for setting value
Definition: frame.h:52
FSTATIC Frame * compressframe_tlvconstructor(gpointer tlvstart, gconstpointer pktend, gpointer *newpacket, gpointer *newpacketend)
#define CASTTOCLASS(Cclass, obj)
Safely cast 'obj' to C-class 'class' - verifying that it was registerd as being of type class ...
Definition: proj_classes.h:66
#define NEWSUBCLASS(Cclass, obj)
Definition: proj_classes.h:67
#define MAXUDPSIZE
Maximum UDP packet size.
Definition: compressframe.h:33