The Assimilation Project
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
compressframe.c
Go to the documentation of this file.
1 
27 
33 
35 #define KBYTES(n) ((n)*1024)
36 #define MAXUNCOMPRESSEDSIZE KBYTES(1024)
39 
40 #include <glib.h>
41 #include <fcntl.h>
42 #include <memory.h>
43 #include <malloc.h>
44 #include <compressframe.h>
45 #include <frameset.h>
46 #include <generic_tlv_min.h>
47 #include <tlvhelper.h>
48 
49 #ifdef HAVE_ZLIB_H
50 # define ZLIB_CONST 1 /* Enable const definitions where appropriate */
51 # include <zlib.h>
52 #endif /* HAVE_ZLIB_H */
53 #if ZLIB_VER_MAJOR > 1 || ZLIB_VER_MINOR > 2 || ZLIB_VER_REVISION > 7
54  // At least version 1.2.8 has the ZLIB_CONST #define to fix this
55 # define ZLIB_CONSTANT(e) ((const guint8*)(e))
56 #else
57  // Older zlib header files didn't declare things const like they should
58  // So we use a sleazy trick to deal with the broken header file :-(
59 # define ZLIB_CONSTANT(e) ((guint8*)(GSIZE_TO_POINTER(GPOINTER_TO_SIZE(e))))
60 #endif
61 
63 
65 FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify);
66 FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet*);
67 FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer);
68 FSTATIC int _compressframe_findmethod(int method);
69 FSTATIC gchar * _compressframe_toString(gconstpointer);
70 FSTATIC void assim_dump_bytes(char * prefix, gconstpointer p, int len);
71 
72 #define COMPFRAMESIZE 4 // one-byte compression type + 3 byte uncompressed length
73 
74 #ifdef HAVE_ZLIB_H
75 FSTATIC gpointer z_compressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
76 FSTATIC gpointer z_decompressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
77 #endif /* HAVE_ZLIB_H */
78 
80 static struct compression_types {
81  guint8 compression_type;
82  gpointer (*compress)(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
84  gpointer (*decompress)(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
86  const char* name;
87 }allcompressions [] = {
88 #ifdef HAVE_ZLIB_H
89  {COMPRESS_ZLIB, z_compressbuf, z_decompressbuf, "zlib"},
90 #endif
91 };
92 
93 #include <stdio.h>
94 FSTATIC void
95 assim_dump_bytes(char * prefix, gconstpointer p, int len)
96 {
97  int j;
98  const char * space = "";
99  fprintf(stderr, "%s", prefix);
100 
101  for (j=0; j < len; ++j) {
102  fprintf(stderr, "%s%02x", space, *(((const guint8*)p)+j));
103  space=" ";
104  }
105  fprintf(stderr, " [%d bytes]\n", len);
106 }
107 FSTATIC int
109 {
110  gsize j;
111  for (j=0; j < DIMOF(allcompressions); ++j) {
112  if (method == allcompressions[j].compression_type) {
113  return j;
114  }
115  }
116  return -1;
117 }
118 
119 //static void(*parentfinalize)(AssimObj*) = NULL;
121 compressframe_new(guint16 frame_type, guint16 compression_method)
122 {
123  Frame* fself;
124  CompressFrame* self;
125  int cmpindex;
126 
128 
129  if ((cmpindex = _compressframe_findmethod(compression_method)) < 0) {
130  g_warning("%s.%d: Unknown compression type: %d"
131  , __FUNCTION__, __LINE__, compression_method);
132  return(NULL);
133  }
134  fself = frame_new(frame_type, sizeof(CompressFrame));
135  self = NEWSUBCLASS(CompressFrame, fself);
136  self->compression_method = compression_method;
137  self->compression_index = cmpindex;
138  self->compression_threshold = DEFAULT_COMPRESSION_THRESHOLD;
139  fself->length = COMPFRAMESIZE;
144 #if 0
145  if (NULL == parentfinalize) {
146  parentfinalize = fself->baseclass._finalize;
147  }
149 #endif
150  return self;
151 }
152 
154 compressframe_new_string(guint16 frame_type, const char* compression_name)
155 {
156  gsize j;
157  for (j=0; j < DIMOF(allcompressions); ++j) {
158  if (strcmp(compression_name, allcompressions[j].name) == 0) {
159  return compressframe_new(frame_type, allcompressions[j].compression_type);
160  }
161  }
162  g_return_val_if_reached(NULL);
163 }
165 FSTATIC gboolean
166 _compressframe_isvalid(const Frame *fself, gconstpointer tlvstart, gconstpointer pktend)
167 {
168  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, fself);
169  guint8 compresstype;
170  guint32 origlen;
171  const guint8* valptr;
172  if (NULL == tlvstart) {
173  return ((gsize)self->compression_index) < DIMOF(allcompressions)
174  && allcompressions[self->compression_index].compression_type == self->compression_method;
175  }
176 
177  if ( ((const guint8*)pktend-(const guint8*)tlvstart) < 12
178  || get_generic_tlv_len(tlvstart, pktend) <= 8) {
179  return FALSE;
180  }
181  valptr = get_generic_tlv_value(tlvstart, pktend);
182  compresstype = tlv_get_guint8(valptr, pktend);
183  g_return_val_if_fail(_compressframe_findmethod(compresstype) >= 0, FALSE);
184  origlen = tlv_get_guint24(valptr+1, pktend);
185  // Trying to avoid a DOS attack using huge packets
186  g_return_val_if_fail(origlen <= MAXUNCOMPRESSEDSIZE || origlen >= 16, FALSE);
187  return TRUE;
188 }
189 
190 FSTATIC void
191 _compressframe_setvalue(Frame *f, gpointer value, guint32 len, GDestroyNotify valnotify)
192 {
193  (void)f;
194  (void)value;
195  (void)len;
196  (void)valnotify;
197  g_warning("%s:%d: Not possible to set the value of a CompressFrame", __FUNCTION__, __LINE__);
198 }
199 
200 FSTATIC gchar *
201 _compressframe_toString(gconstpointer aself)
202 {
203  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, aself);
204  double ratio;
205  if (self->baseclass.length <= 4) {
206  return g_strdup_printf("CompressFrame(frametype:%d, method:%s)"
207  , self->baseclass.type, allcompressions[self->compression_index].name);
208  }
209  ratio = (double)self->decompressed_size / ((double)(self->baseclass.length-4));
210  return g_strdup_printf("CompressFrame(frametype:%d, method:%s, len:%d uncompsize:%d, ratio:%0.2f:1)"
211  , self->baseclass.type, allcompressions[self->compression_index].name
212  , self->baseclass.length, self->decompressed_size, ratio);
213 }
214 
215 
221 FSTATIC void
222 _compressframe_updatedata(Frame *f, //<[in] Our frame ("self")
223  gpointer tlvstart, //<[in] Beginning of where to put our data (not really)
224  gconstpointer pktend, //<[out] Where to put the final data (not really)
225  FrameSet* fs) //<[in/out] Frameset to update the "packet" for
226  //< once we compress this data
227 {
229  guint8* pktstart = fs->packet;
230  guint8* tlvstart8 = tlvstart;
231  const guint8* pktend8 = pktend;
232  guint8* valptr;
233  guint32 ouroffset; // Offset to beginning of our packet
234  guint32 cmpoffset; // Offset to beginning of compressed data
235  guint8* newpacket;
236  guint8* newpktend;
237  int compressedsize;
238 
239 
240  // Now on to our side effect - compressing the frames that follow us...
241  ouroffset = (tlvstart8-pktstart);
242  cmpoffset = ouroffset + COMPFRAMESIZE+FRAME_INITSIZE;
243  newpacket = allcompressions[self->compression_index].compress
244  (pktstart, pktend8-pktstart, cmpoffset, MAXUDPSIZE, &compressedsize, 0);
245  self->decompressed_size = (pktend8 - pktstart) - cmpoffset;
246 
247  if (NULL == newpacket) {
248  g_warning("%s:%d: Unable to compress %d byte packet to %d byte UDP packet"
249  , __FUNCTION__, __LINE__, self->decompressed_size, MAXUDPSIZE);
250  }
251  newpktend = newpacket + compressedsize;
252  // Write our type and length into the (new) packet
253  set_generic_tlv_type(newpacket+ouroffset, f->type, newpktend);
254  self->baseclass.length = (compressedsize-cmpoffset) + COMPFRAMESIZE;
255  set_generic_tlv_len(newpacket+ouroffset, self->baseclass.length, newpktend);
256  valptr = get_generic_tlv_nonconst_value(newpacket+ouroffset, newpktend);
257  // Our TLV value consists of the compression method followed by a 3 byte
258  // packet length, followed by the compressed data (already in "newpacket").
259  // This restricts us to a 16M decompressed original packet.
260  // Since this has to compress down to a single UDP packet,
261  // this a very reasonable assumption...
262  // In practice, our JSON seems to be limited to about 300K decompressed.
263  tlv_set_guint8(valptr, self->compression_method, newpktend);
264  tlv_set_guint24(valptr+1, self->decompressed_size, newpktend);
265  if (fs->packet) {
266  FREE(fs->packet);
267  }
268  fs->packet = newpacket;
269  fs->pktend = newpktend;
270 }
271 
272 #define COMPRESSFRAMEMIN 4
273 FSTATIC Frame*
274 compressframe_tlvconstructor(gpointer tlvstart,
275  gconstpointer pktend,
276  gpointer* newpacket,
277  gpointer* newpacketend)
278 {
279  const guint8* pktend8 = pktend;
280  const guint8* tlvstart8 = tlvstart;
281  const guint8* packet;
282  const guint8* valueptr;
283  guint32 cmppktsize;
284  guint8 compression_type;
285  guint32 decompressed_size;
286  int actual_size;
287  int compression_index;
288  guint16 frametype;
290  CompressFrame* ret;
291  /* Our four bytes of real data are:
292  * 1-byte compression type
293  * 3-byte decompressed size
294  */
295  frametype = get_generic_tlv_type(tlvstart, pktend);
296  valueptr = get_generic_tlv_value(tlvstart, pktend);
297  compression_type = tlv_get_guint8(valueptr, pktend);
298  decompressed_size = tlv_get_guint24(valueptr+1, pktend);
299  compression_index = _compressframe_findmethod(compression_type);
300  // Trying to mitigate possible DOS attack using huge packets
301  // In practice, our max JSON decompressed size is under 325K
302  g_return_val_if_fail(decompressed_size <= MAXUNCOMPRESSEDSIZE, NULL);
303  g_return_val_if_fail(decompressed_size > 16, NULL);
304  g_return_val_if_fail(compression_index >= 0, NULL);
305  packet = valueptr + COMPRESSFRAMEMIN;
306 
307  cmppktsize = pktend8 - tlvstart8; // Compressed packet size
308 
309  *newpacket = allcompressions[compression_index].decompress
310  (packet, cmppktsize, 0, decompressed_size, &actual_size);
311  g_return_val_if_fail(*newpacket != NULL, NULL);
312  *newpacketend = (guint8*)*newpacket + actual_size;
313 
314  ret = compressframe_new(frametype, compression_type);
315  g_return_val_if_fail(ret != NULL, NULL);
316  ret->decompressed_size = decompressed_size;
317  return &ret->baseclass;
318 }
319 
320 #ifdef HAVE_ZLIB_H
321 gpointer
327 z_compressbuf(gconstpointer inbuf
328 , int insize
329 , int offset
330 , int maxout
331 , int *actualsize
332 , int level)
333 {
334 
335  guint8* outbuf;
336  z_stream stream;
337  int ret;
338 #if 0
339  int space;
340  double ratio;
341 #endif
342 
343  // Compute compression level
344  // If our guess doesn't work, we'll escalate to max compression
345  // This adds compression expense, this is mostly in the nanoprobe side, so we don't much care.
346  if (level < 1) {
347  if (insize < KBYTES(189)) {
348  level = 1;
349  }else if (insize < KBYTES(225)) {
350  level = 6;
351  }else{
352  level = 9;
353  }
354  }
355 
356  /* Set up libz */
357  stream.zalloc = Z_NULL;
358  stream.zfree = Z_NULL;
359  stream.opaque = Z_NULL;
360  ret = deflateInit(&stream, level);
361 
362  if (ret != Z_OK) {
363  g_warning("%s.%d: OOPS, got a deflateInit return of %d"
364  , __FUNCTION__, __LINE__, ret);
365  return NULL;
366  }
367 #if 0
368  outbuf = g_malloc(maxout);
369 #else
370  outbuf = calloc(maxout, 1);
371 #endif
372  if (NULL == outbuf) {
373  g_warning("%s.%d: OOPS out of space.", __FUNCTION__, __LINE__);
374  return NULL;
375  }
376  stream.avail_in = insize-offset;
377  stream.next_in = ZLIB_CONSTANT(inbuf)+offset;
378  stream.avail_out = maxout-offset;
379  stream.next_out = outbuf+offset;
380 
381  /* Compress it */
382  ret = deflate(&stream, Z_FINISH);
383  if (ret != Z_STREAM_END) {
384  g_free(outbuf);
385  if (level < 9) {
386  return z_compressbuf(inbuf, insize, offset, maxout, actualsize, 9);
387  }
388  g_warning("%s.%d: Got return code %d from deflate."
389  , __FUNCTION__, __LINE__, ret);
390  return NULL;
391  }
392 #if 0
393  {
394  double ratio;
395  ratio = ((double)stream.total_in/((double)stream.total_out));
396  fprintf(stderr, "Compressing %ld bytes into %ld bytes with level %d ratio %.2f:1\n"
397  , stream.total_in, stream.total_out, level, ratio);
398  }
399 #endif
400  (void)deflateEnd(&stream);
401  *actualsize = stream.total_out + offset;
402  outbuf = g_realloc(outbuf, *actualsize);
403  return outbuf;
404 }
405 
408 gpointer
409 z_decompressbuf(gconstpointer inbuf
410 , int insize
411 , int offset
412 , int maxout
414 , int *uncompsize)
416 {
417  gpointer outbuf;
418  int outsize;
419  int ret;
420  z_stream stream;
421  /* Set up libz */
422  stream.zalloc = Z_NULL;
423  stream.zfree = Z_NULL;
424  stream.opaque = Z_NULL;
425  stream.avail_in = insize-offset;
426  stream.next_in = ZLIB_CONSTANT(inbuf) + offset;
427  g_return_val_if_fail (Z_OK == inflateInit(&stream), NULL);
428  outbuf = g_malloc(maxout);
429  if (offset > 0) {
430  memcpy(outbuf, inbuf, offset);
431  }
432  stream.avail_out = maxout;
433  stream.next_out = ((guint8 *)outbuf) + offset;
434  // Decompress our input buffer.
435  ret = inflate(&stream, Z_FINISH);
436  (void)inflateEnd(&stream);
437  if (ret != Z_STREAM_END) {
438  g_warning( "%s.%d: GOT inflate RETURN OF %d"
439  , __FUNCTION__, __LINE__, ret);
440  g_free(outbuf);
441  return NULL;
442  }
443  outsize = maxout - stream.avail_out;
444  if (outsize > maxout) {
445  outbuf = g_realloc(outbuf, outsize);
446  }
447  (void)inflateEnd(&stream);
448  *uncompsize = outsize;
449  return outbuf;
450 }
451 #endif /* HAVE_ZLIB_H */
#define FRAME_INITSIZE
Definition: frame.h:59
guint32 length
Frame Length.
Definition: frame.h:46
FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint32, GDestroyNotify valnotify)
guint8 tlv_get_guint8(const void *vitem, const void *bufend)
Retrieve an unsigned 8 bit integer from the given location with error checking.
Definition: tlvhelper.c:31
AssimObj baseclass
Base object class for our Class system.
Definition: frame.h:44
Implements minimal client-oriented Frame and Frameset capabilities.
This is the base Frame class object (in-memory TLV (type, length, value)) for every general component...
Definition: frame.h:43
#define ZLIB_CONSTANT(e)
Definition: compressframe.c:59
gpointer pktend
Last byte past the end of the packet.
Definition: frameset.h:51
#define FSTATIC
Definition: projectcommon.h:31
#define KBYTES(n)
Definition: compressframe.c:35
FSTATIC void _compressframe_finalize(AssimObj *self)
void tlv_set_guint24(void *vitem, guint32 item, const void *bufend)
Set an unsigned 24 bit (3-byte) integer to the given location with error checking and without caring ...
Definition: tlvhelper.c:148
guint16 get_generic_tlv_type(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Type' of the given TLV TLV entry (first two bytes)
#define BINDDEBUG(Cclass)
BINDDEBUG is for telling the class system where the debug variable for this class is - put it in the ...
Definition: proj_classes.h:82
#define __FUNCTION__
CompressFrame * compressframe_new(guint16 frame_type, guint16 compression_method)
This is our CompressFrame class object - used for representing a compression method.
Definition: compressframe.h:41
#define MAXUNCOMPRESSEDSIZE
In practice, our max JSON decompressed size is under 325K, so 1M seems safe more or less not matter w...
Definition: compressframe.c:38
Frame * frame_new(guint16 frame_type, gsize framesize)
Construct a new frame - allowing for "derived" frame types...
Definition: frame.c:125
#define COMPRESS_ZLIB
Compression using 'zlib'.
Definition: compressframe.h:38
gboolean(* isvalid)(const Frame *self, gconstpointer tlvptr, gconstpointer pktend)
TRUE if TLV data looks valid...
Definition: frame.h:50
Describes interfaces to C-String Frame (Compressframe) C-Class It holds conventional zero-terminated ...
#define COMPFRAMESIZE
Definition: compressframe.c:72
#define FREE(m)
Our interface to free.
Definition: projectcommon.h:29
gpointer packet
Pointer to packet (when constructed)
Definition: frameset.h:50
guint32 tlv_get_guint24(const void *vitem, const void *bufend)
Retrieve an unsigned 24 bit (3-byte) integer from the given location with error checking and without ...
Definition: tlvhelper.c:133
#define CASTTOCONSTCLASS(Cclass, obj)
Safely cast 'obj' to const C-class 'class' - verifying that it was registered as being of type class ...
Definition: proj_classes.h:71
void set_generic_tlv_type(gpointer tlv_vp, guint16 newtype, gconstpointer pktend)
Set the 'Type' of the given TLV TLV entry (first two bytes)
guint32 get_generic_tlv_len(gconstpointer tlv_vp, gconstpointer pktend)
Return the 'Length' of the given generic TLV entry (first 3 bytes after type) Note: return result is ...
#define DIMOF(a)
Definition: lldp_min.c:30
TLV helper interfaces definitions.
void tlv_set_guint8(void *vitem, guint8 item, const void *bufend)
Set an unsigned 8 bit integer to the given location with error checking.
Definition: tlvhelper.c:42
#define DEFAULT_COMPRESSION_THRESHOLD
Default value of when to start compressing.
Definition: compressframe.h:34
guint32 decompressed_size
Definition: compressframe.h:44
#define COMPRESSFRAMEMIN
void set_generic_tlv_len(gpointer tlv_vp, guint32 newsize, gconstpointer pktend)
Set the 'Length' of the given generic TLV entry (first 3 bytes after type)
void(* _finalize)(AssimObj *)
Free object (private)
Definition: assimobj.h:55
guint16 type
Frame Type (see Individual TLV 'Frame' data types and layouts (by TLV type) - frameformats.h )
Definition: frame.h:45
Provides definitions for using our generic TLV capabilities.
FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer)
Return TRUE if this is a valid CompressFrame - either an object or on-the-wire version.
gconstpointer get_generic_tlv_value(gconstpointer tlv_vp, gconstpointer pktend)
Return a const pointer to the 'Value' of the given generic TLV entry.
DEBUGDECLARATIONS
Definition: compressframe.c:62
FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet *)
We update the data in the packet from our CompressFrame object AND ALSO have the side-effect of compr...
FSTATIC void assim_dump_bytes(char *prefix, gconstpointer p, int len)
Definition: compressframe.c:95
FSTATIC gchar * _compressframe_toString(gconstpointer)
void(* updatedata)(Frame *self, gpointer tlvptr, gconstpointer pktend, FrameSet *fs)
Update packet data.
Definition: frame.h:49
FrameSet class - used for collecting Frames when not on the wire, and for marshalling/demarshalling t...
Definition: frameset.h:46
gchar *(* toString)(gconstpointer)
Produce malloc-ed string representation.
Definition: assimobj.h:58
FSTATIC int _compressframe_findmethod(int method)
gpointer get_generic_tlv_nonconst_value(gpointer tlv_vp, gconstpointer pktend)
Return a non-const (mutable) pointer to the 'Value' of the given generic TLV entry.
CompressFrame * compressframe_new_string(guint16 frame_type, const char *compression_name)
void(* setvalue)(Frame *self, gpointer value, guint32 length, GDestroyNotify valfinal)
member function for setting value
Definition: frame.h:52
FSTATIC Frame * compressframe_tlvconstructor(gpointer tlvstart, gconstpointer pktend, gpointer *newpacket, gpointer *newpacketend)
#define CASTTOCLASS(Cclass, obj)
Safely cast 'obj' to C-class 'class' - verifying that it was registerd as being of type class ...
Definition: proj_classes.h:66
#define NEWSUBCLASS(Cclass, obj)
Definition: proj_classes.h:67
#define MAXUDPSIZE
Maximum UDP packet size.
Definition: compressframe.h:33