The Assimilation Project
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
compressframe.c
Go to the documentation of this file.
1 
27 
33 
35 #define KBYTES(n) ((n)*1024)
36 #define MAXUNCOMPRESSEDSIZE KBYTES(1024)
39 
40 #include <glib.h>
41 #include <fcntl.h>
42 #include <memory.h>
43 #include <malloc.h>
44 #include <compressframe.h>
45 #include <frameset.h>
46 #include <generic_tlv_min.h>
47 #include <tlvhelper.h>
48 
49 #ifdef HAVE_ZLIB_H
50 # define ZLIB_CONST 1 /* Enable const definitions where appropriate */
51 # include <zlib.h>
52 #endif /* HAVE_ZLIB_H */
53 #if ZLIB_VER_MAJOR > 1 || ZLIB_VER_MINOR > 2 || ZLIB_VER_REVISION > 7
54  // At least version 1.2.8 has the ZLIB_CONST #define to fix this
55 # define ZLIB_CONSTANT(e) ((const guint8*)(e))
56 #else
57  // Older zlib header files didn't declare things const like they should
58  // So we use a sleazy trick to deal with the broken header file :-(
59 # define ZLIB_CONSTANT(e) ((guint8*)(GSIZE_TO_POINTER(GPOINTER_TO_SIZE(e))))
60 #endif
61 
63 
65 FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify);
66 FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet*);
67 FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer);
68 FSTATIC int _compressframe_findmethod(int method);
69 FSTATIC gchar * _compressframe_toString(gconstpointer);
70 FSTATIC void assim_dump_bytes(char * prefix, gconstpointer p, int len);
71 
72 #ifdef HAVE_ZLIB_H
73 FSTATIC gpointer z_compressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
74 FSTATIC gpointer z_decompressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
75 #endif /* HAVE_ZLIB_H */
76 
78 static struct compression_types {
79  guint8 compression_type;
80  gpointer (*compress)(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
82  gpointer (*decompress)(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
84  const char* name;
85 }allcompressions [] = {
86 #ifdef HAVE_ZLIB_H
87  {COMPRESS_ZLIB, z_compressbuf, z_decompressbuf, "zlib"},
88 #endif
89 };
90 
91 #include <stdio.h>
92 FSTATIC void
93 assim_dump_bytes(char * prefix, gconstpointer p, int len)
94 {
95  int j;
96  const char * space = "";
97  fprintf(stderr, "%s", prefix);
98 
99  for (j=0; j < len; ++j) {
100  fprintf(stderr, "%s%02x", space, *(((const guint8*)p)+j));
101  space=" ";
102  }
103  fprintf(stderr, " [%d bytes]\n", len);
104 }
105 FSTATIC int
107 {
108  gsize j;
109  for (j=0; j < DIMOF(allcompressions); ++j) {
110  if (method == allcompressions[j].compression_type) {
111  return j;
112  }
113  }
114  return -1;
115 }
116 
117 //static void(*parentfinalize)(AssimObj*) = NULL;
119 compressframe_new(guint16 frame_type, guint16 compression_method)
120 {
121  Frame* fself;
122  CompressFrame* self;
123  int cmpindex;
124 
126 
127  if ((cmpindex = _compressframe_findmethod(compression_method)) < 0) {
128  g_warning("%s.%d: Unknown compression type: %d"
129  , __FUNCTION__, __LINE__, compression_method);
130  return(NULL);
131  }
132  fself = frame_new(frame_type, sizeof(CompressFrame));
133  self = NEWSUBCLASS(CompressFrame, fself);
134  self->compression_method = compression_method;
135  self->compression_index = cmpindex;
136  self->compression_threshold = DEFAULT_COMPRESSION_THRESHOLD;
137  fself->length = 4;
142 #if 0
143  if (NULL == parentfinalize) {
144  parentfinalize = fself->baseclass._finalize;
145  }
147 #endif
148  return self;
149 }
150 
152 compressframe_new_string(guint16 frame_type, const char* compression_name)
153 {
154  gsize j;
155  for (j=0; j < DIMOF(allcompressions); ++j) {
156  if (strcmp(compression_name, allcompressions[j].name) == 0) {
157  return compressframe_new(frame_type, allcompressions[j].compression_type);
158  }
159  }
160  g_return_val_if_reached(NULL);
161 }
163 FSTATIC gboolean
164 _compressframe_isvalid(const Frame *fself, gconstpointer tlvstart, gconstpointer pktend)
165 {
166  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, fself);
167  guint8 compresstype;
168  guint32 origlen;
169  const guint8* valptr;
170  if (tlvstart == NULL) {
171  return ((gsize)self->compression_index) < DIMOF(allcompressions)
172  && allcompressions[self->compression_index].compression_type == self->compression_method;
173  }
174 
175  if ( ((const guint8*)pktend-(const guint8*)tlvstart) < 12
176  || get_generic_tlv_len(tlvstart, pktend) <= 8) {
177  return FALSE;
178  }
179  valptr = get_generic_tlv_value(tlvstart, pktend);
180  compresstype = tlv_get_guint8(valptr, pktend);
181  g_return_val_if_fail(_compressframe_findmethod(compresstype) >= 0, FALSE);
182  origlen = tlv_get_guint24(valptr+1, pktend);
183  // Trying to avoid a DOS attack using huge packets
184  g_return_val_if_fail(origlen <= MAXUNCOMPRESSEDSIZE || origlen >= 16, FALSE);
185  return TRUE;
186 }
187 
188 FSTATIC void
189 _compressframe_setvalue(Frame *f, gpointer value, guint32 len, GDestroyNotify valnotify)
190 {
191  (void)f;
192  (void)value;
193  (void)len;
194  (void)valnotify;
195  g_warning("%s:%d: Not possible to set the value of a CompressFrame", __FUNCTION__, __LINE__);
196 }
197 
198 FSTATIC gchar *
199 _compressframe_toString(gconstpointer aself)
200 {
201  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, aself);
202  double ratio;
203  if (self->baseclass.length <= 4) {
204  return g_strdup_printf("CompressFrame(frametype:%d, method:%s)"
205  , self->baseclass.type, allcompressions[self->compression_index].name);
206  }
207  ratio = (double)self->decompressed_size / ((double)(self->baseclass.length-4));
208  return g_strdup_printf("CompressFrame(frametype:%d, method:%s, len:%d uncompsize:%d, ratio:%0.2f:1)"
209  , self->baseclass.type, allcompressions[self->compression_index].name
210  , self->baseclass.length, self->decompressed_size, ratio);
211 }
212 
213 #define COMPFRAMESIZE 4 // one-byte compression type + 3 byte uncompressed length
214 
220 FSTATIC void
221 _compressframe_updatedata(Frame *f, //<[in] Our frame ("self")
222  gpointer tlvstart, //<[in] Beginning of where to put our data (not really)
223  gconstpointer pktend, //<[out] Where to put the final data (not really)
224  FrameSet* fs) //<[in/out] Frameset to update the "packet" for
225  //< once we compress this data
226 {
228  guint8* pktstart = fs->packet;
229  guint8* tlvstart8 = tlvstart;
230  const guint8* pktend8 = pktend;
231  guint8* valptr;
232  guint32 ouroffset;
233  guint32 cmpoffset;
234  guint8* newpacket;
235  guint8* newpktend;
236  int compressedsize;
237 
238 
239  // Now on to our side effect - compressing the frames that follow us...
240  ouroffset = (tlvstart8-pktstart);
241  cmpoffset = ouroffset + COMPFRAMESIZE+FRAME_INITSIZE;
242  newpacket = allcompressions[self->compression_index].compress
243  (pktstart, pktend8-pktstart, cmpoffset, MAXUDPSIZE, &compressedsize, 0);
244  self->decompressed_size = (pktend8 - pktstart) - cmpoffset;
245 {
246  char * frameset = fs->baseclass.toString(fs);
247  g_free(frameset); frameset = NULL;
248 }
249  if (NULL == newpacket) {
250  g_warning("%s:%d: Unable to compress %d byte packet to %d byte UDP packet"
251  , __FUNCTION__, __LINE__, self->decompressed_size, MAXUDPSIZE);
252  }
253  newpktend = newpacket + compressedsize;
254  // Write our type and length into the (new) packet
255  set_generic_tlv_type(newpacket+ouroffset, f->type, newpktend);
256  self->baseclass.length = (compressedsize-cmpoffset) + COMPFRAMESIZE;
257  set_generic_tlv_len(newpacket+ouroffset, self->baseclass.length, newpktend);
258  valptr = get_generic_tlv_nonconst_value(newpacket+ouroffset, newpktend);
259  // Our TLV value consists of the compression method followed by a 3 byte
260  // packet length, followed by the compressed data (already in "newpacket").
261  // This restricts us to a 16M decompressed original packet.
262  // Since this has to compress down to a single UDP packet,
263  // this a very reasonable assumption...
264  // In practice, our JSON seems to be limited to about 300K decompressed.
265  tlv_set_guint8(valptr, self->compression_method, newpktend);
266  tlv_set_guint24(valptr+1, self->decompressed_size, newpktend);
267  fs->packet = newpacket;
268  fs->pktend = newpktend;
269 }
270 
271 #define COMPRESSFRAMEMIN 4
272 FSTATIC Frame*
273 compressframe_tlvconstructor(gconstpointer tlvstart,
274  gconstpointer pktend,
275  gpointer* newpacket,
276  gpointer* newpacketend)
277 {
278  const guint8* pktend8 = pktend;
279  const guint8* tlvstart8 = tlvstart;
280  const guint8* packet;
281  const guint8* valueptr;
282  guint32 cmppktsize;
283  guint8 compression_type;
284  guint32 decompressed_size;
285  int actual_size;
286  int compression_index;
287  guint16 frametype;
289  CompressFrame* ret;
290  /* Our four bytes of real data are:
291  * 1-byte compression type
292  * 3-byte decompressed size
293  */
294  frametype = get_generic_tlv_type(tlvstart, pktend);
295  valueptr = get_generic_tlv_value(tlvstart, pktend);
296  compression_type = tlv_get_guint8(valueptr, pktend);
297  decompressed_size = tlv_get_guint24(valueptr+1, pktend);
298  compression_index = _compressframe_findmethod(compression_type);
299  // Trying to mitigate possible DOS attack using huge packets
300  // In practice, our max JSON decompressed size is under 325K
301  g_return_val_if_fail(decompressed_size <= MAXUNCOMPRESSEDSIZE, NULL);
302  g_return_val_if_fail(decompressed_size > 16, NULL);
303  g_return_val_if_fail(compression_index >= 0, NULL);
304  packet = valueptr + COMPRESSFRAMEMIN;
305 
306  cmppktsize = pktend8 - tlvstart8; // Compressed packet size
307 
308  *newpacket = allcompressions[compression_index].decompress
309  (packet, cmppktsize, 0, decompressed_size, &actual_size);
310  g_return_val_if_fail(*newpacket != NULL, NULL);
311  *newpacketend = (guint8*)*newpacket + actual_size;
312 
313  ret = compressframe_new(frametype, compression_type);
314  g_return_val_if_fail(ret != NULL, NULL);
315  ret->decompressed_size = decompressed_size;
316  return &ret->baseclass;
317 }
318 
319 #ifdef HAVE_ZLIB_H
320 gpointer
326 z_compressbuf(gconstpointer inbuf
327 , int insize
328 , int offset
329 , int maxout
330 , int *actualsize
331 , int level)
332 {
333 
334  guint8* outbuf;
335  z_stream stream;
336  int ret;
337 #if 0
338  int space;
339  double ratio;
340 #endif
341 
342  // Compute compression level
343  // If our guess doesn't work, we'll escalate to max compression
344  // This adds compression expense, this is mostly in the nanoprobe side, so we don't much care.
345  if (level < 1) {
346  if (insize < KBYTES(189)) {
347  level = 1;
348  }else if (insize < KBYTES(225)) {
349  level = 6;
350  }else{
351  level = 9;
352  }
353  }
354 
355  /* Set up libz */
356  stream.zalloc = Z_NULL;
357  stream.zfree = Z_NULL;
358  stream.opaque = Z_NULL;
359  ret = deflateInit(&stream, level);
360 
361  if (ret != Z_OK) {
362  g_warning("%s.%d: OOPS, got a deflateInit return of %d"
363  , __FUNCTION__, __LINE__, ret);
364  return NULL;
365  }
366 #if 0
367  outbuf = g_malloc(maxout);
368 #else
369  outbuf = calloc(maxout, 1);
370 #endif
371  if (NULL == outbuf) {
372  g_warning("%s.%d: OOPS out of space.", __FUNCTION__, __LINE__);
373  return NULL;
374  }
375  stream.avail_in = insize-offset;
376  stream.next_in = ZLIB_CONSTANT(inbuf)+offset;
377  stream.avail_out = maxout-offset;
378  stream.next_out = outbuf+offset;
379 
380  /* Compress it */
381  ret = deflate(&stream, Z_FINISH);
382  if (ret != Z_STREAM_END) {
383  g_free(outbuf);
384  if (level < 9) {
385  return z_compressbuf(inbuf, insize, offset, maxout, actualsize, 9);
386  }
387  g_warning("%s.%d: Got return code %d from deflate."
388  , __FUNCTION__, __LINE__, ret);
389  return NULL;
390  }
391 #if 0
392  space = insize - stream.avail_out;
393  ratio = ((double)stream.total_in/((double)stream.total_out));
394  //return 0;
395  fprintf(stderr, "Compressing %ld bytes into %ld bytes with level %d ratio %.2f:1\n", stream.total_in
396  , stream.total_out, level, ratio);
397 #endif
398  (void)deflateEnd(&stream);
399  *actualsize = stream.total_out + offset;
400  outbuf = g_realloc(outbuf, *actualsize);
401  return outbuf;
402 }
403 
406 gpointer
407 z_decompressbuf(gconstpointer inbuf
408 , int insize
409 , int offset
410 , int maxout
412 , int *uncompsize)
414 {
415  gpointer outbuf;
416  int outsize;
417  int ret;
418  z_stream stream;
419  /* Set up libz */
420  stream.zalloc = Z_NULL;
421  stream.zfree = Z_NULL;
422  stream.opaque = Z_NULL;
423  stream.avail_in = insize-offset;
424  stream.next_in = ZLIB_CONSTANT(inbuf) + offset;
425  g_return_val_if_fail (Z_OK == inflateInit(&stream), NULL);
426  outbuf = g_malloc(maxout);
427  if (offset > 0) {
428  memcpy(outbuf, inbuf, offset);
429  }
430  stream.avail_out = maxout;
431  stream.next_out = ((guint8 *)outbuf) + offset;
432  // Decompress our input buffer.
433  ret = inflate(&stream, Z_FINISH);
434  (void)inflateEnd(&stream);
435  if (ret != Z_STREAM_END) {
436  g_warning( "%s.%d: GOT inflate RETURN OF %d"
437  , __FUNCTION__, __LINE__, ret);
438  g_free(outbuf);
439  return NULL;
440  }
441  outsize = maxout - stream.avail_out;
442  if (outsize > maxout) {
443  outbuf = g_realloc(outbuf, outsize);
444  }
445  (void)inflateEnd(&stream);
446  *uncompsize = outsize;
447  return outbuf;
448 }
449 #endif /* HAVE_ZLIB_H */
#define FRAME_INITSIZE
Definition: frame.h:59
AssimObj baseclass
Definition: frameset.h:46
guint32 length
Frame Length.
Definition: frame.h:46
FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify)
guint8 tlv_get_guint8(const void *vitem, const void *bufend)
Retrieve an unsigned 8 bit integer from the given location with error checking.
Definition: tlvhelper.c:30
AssimObj baseclass
Base object class for our Class system.
Definition: frame.h:44
Implements minimal client-oriented Frame and Frameset capabilities.
This is the base Frame class object (in-memory TLV (type, length, value)) for every general component...
Definition: frame.h:43
#define ZLIB_CONSTANT(e)
Definition: compressframe.c:59
gpointer pktend
Last byte past the end of the packet.
Definition: frameset.h:50
#define FSTATIC
Definition: projectcommon.h:31
#define KBYTES(n)
Definition: compressframe.c:35
FSTATIC void _compressframe_finalize(AssimObj *self)
void tlv_set_guint24(void *vitem, guint32 item, const void *bufend)
Set an unsigned 24 bit (3-byte) integer to the given location with error checking and without caring ...
Definition: tlvhelper.c:146
guint16 get_generic_tlv_type(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Type' of the given TLV TLV entry (first two bytes)
#define BINDDEBUG(Cclass)
BINDDEBUG is for telling the class system where the debug variable for this class is - put it in the ...
Definition: proj_classes.h:82
#define __FUNCTION__
CompressFrame * compressframe_new(guint16 frame_type, guint16 compression_method)
This is our CompressFrame class object - used for representing a compression method.
Definition: compressframe.h:41
#define MAXUNCOMPRESSEDSIZE
In practice, our max JSON decompressed size is under 325K, so 1M seems safe more or less not matter w...
Definition: compressframe.c:38
Frame * frame_new(guint16 frame_type, gsize framesize)
Construct a new frame - allowing for "derived" frame types...
Definition: frame.c:124
#define COMPRESS_ZLIB
Compression using 'zlib'.
Definition: compressframe.h:38
gboolean(* isvalid)(const Frame *self, gconstpointer tlvptr, gconstpointer pktend)
TRUE if TLV data looks valid...
Definition: frame.h:50
Describes interfaces to C-String Frame (Compressframe) C-Class It holds conventional zero-terminated ...
#define COMPFRAMESIZE
gpointer packet
Pointer to packet (when constructed)
Definition: frameset.h:49
guint32 tlv_get_guint24(const void *vitem, const void *bufend)
Retrieve an unsigned 24 bit (3-byte) integer from the given location with error checking and without ...
Definition: tlvhelper.c:131
#define CASTTOCONSTCLASS(Cclass, obj)
Safely cast 'obj' to const C-class 'class' - verifying that it was registered as being of type class ...
Definition: proj_classes.h:71
void set_generic_tlv_type(gpointer tlv_vp, guint16 newtype, gconstpointer pktend)
Set the 'Type' of the given TLV TLV entry (first two bytes)
guint32 get_generic_tlv_len(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Length' of the given generic TLV entry (first 3 bytes after type)
#define DIMOF(a)
Definition: lldp_min.c:30
TLV helper interfaces definitions.
void tlv_set_guint8(void *vitem, guint8 item, const void *bufend)
Set an unsigned 8 bit integer to the given location with error checking.
Definition: tlvhelper.c:41
#define DEFAULT_COMPRESSION_THRESHOLD
Default value of when to start compressing.
Definition: compressframe.h:34
guint32 decompressed_size
Definition: compressframe.h:44
FSTATIC Frame * compressframe_tlvconstructor(gconstpointer tlvstart, gconstpointer pktend, gpointer *newpacket, gpointer *newpacketend)
#define COMPRESSFRAMEMIN
void set_generic_tlv_len(gpointer tlv_vp, guint32 newsize, gconstpointer pktend)
Set the 'Length' of the given generic TLV entry (first 3 bytes after type)
void(* _finalize)(AssimObj *)
Free object (private)
Definition: assimobj.h:55
guint16 type
Frame Type (see Individual TLV 'Frame' data types and layouts (by TLV type) - frameformats.h )
Definition: frame.h:45
Provides definitions for using our generic TLV capabilities.
FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer)
Return TRUE if this is a valid CompressFrame - either an object or on-the-wire version.
gconstpointer get_generic_tlv_value(gconstpointer tlv_vp, gconstpointer pktend)
Return a const pointer to the 'Value' of the given generic TLV entry.
DEBUGDECLARATIONS
Definition: compressframe.c:62
FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet *)
We update the data in the packet from our CompressFrame object AND ALSO have the side-effect of compr...
FSTATIC void assim_dump_bytes(char *prefix, gconstpointer p, int len)
Definition: compressframe.c:93
FSTATIC gchar * _compressframe_toString(gconstpointer)
void(* updatedata)(Frame *self, gpointer tlvptr, gconstpointer pktend, FrameSet *fs)
Update packet data.
Definition: frame.h:49
FrameSet class - used for collecting Frames when not on the wire, and for marshalling/demarshalling t...
Definition: frameset.h:45
gchar *(* toString)(gconstpointer)
Produce malloc-ed string representation.
Definition: assimobj.h:58
FSTATIC int _compressframe_findmethod(int method)
gpointer get_generic_tlv_nonconst_value(gpointer tlv_vp, gconstpointer pktend)
Return a non-const (mutable) pointer to the 'Value' of the given generic TLV entry.
CompressFrame * compressframe_new_string(guint16 frame_type, const char *compression_name)
void(* setvalue)(Frame *self, gpointer value, guint32 length, GDestroyNotify valfinal)
member function for setting value
Definition: frame.h:52
#define CASTTOCLASS(Cclass, obj)
Safely cast 'obj' to C-class 'class' - verifying that it was registerd as being of type class ...
Definition: proj_classes.h:66
#define NEWSUBCLASS(Cclass, obj)
Definition: proj_classes.h:67
#define MAXUDPSIZE
Maximum UDP packet size.
Definition: compressframe.h:33