The Assimilation Project
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
compressframe.c
Go to the documentation of this file.
1 
27 
33 
35 #define KBYTES(n) ((n)*1024)
36 #define MAXUNCOMPRESSEDSIZE KBYTES(1024)
39 
40 #include <glib.h>
41 #include <fcntl.h>
42 #include <memory.h>
43 #include <compressframe.h>
44 #include <frameset.h>
45 #include <generic_tlv_min.h>
46 #include <tlvhelper.h>
47 
48 #ifdef HAVE_ZLIB_H
49 # define ZLIB_CONST 1 /* Enable const definitions where appropriate */
50 # include <zlib.h>
51 #endif /* HAVE_ZLIB_H */
52 
54 
56 FSTATIC gsize _compressframe_dataspace(const Frame* f);
57 FSTATIC void _compressframe_setvalue(Frame *, gpointer, guint16, GDestroyNotify valnotify);
58 FSTATIC void _compressframe_updatedata(Frame *, gpointer, gconstpointer, FrameSet*);
59 FSTATIC gboolean _compressframe_isvalid(const Frame *, gconstpointer, gconstpointer);
60 FSTATIC int _compressframe_findmethod(int method);
61 
62 #ifdef HAVE_ZLIB_H
63 FSTATIC gpointer z_compressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
64 FSTATIC gpointer z_decompressbuf(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
65 #endif /* HAVE_ZLIB_H */
66 
68 static struct compression_types {
69  guint8 compression_type;
70  gpointer (*compress)(gconstpointer inbuf, int insize, int offset, int maxout, int *actualsize, int level);
72  gpointer (*decompress)(gconstpointer inbuf, int insize, int offset, int maxout, int *uncompsize);
74 }allcompressions [] = {
75 #ifdef HAVE_ZLIB_H
76  {COMPRESS_ZLIB, z_compressbuf, z_decompressbuf},
77 #endif
78 };
79 
80 FSTATIC int
82 {
83  unsigned j;
84  for (j=0; j < DIMOF(allcompressions); ++j) {
85  if (method == allcompressions[j].compression_type) {
86  return j;
87  }
88  }
89  return -1;
90 }
91 
92 //static void(*parentfinalize)(AssimObj*) = NULL;
94 compressframe_new(guint16 frame_type, guint16 compression_method)
95 {
96  Frame* fself;
97  CompressFrame* self;
98  int cmpindex;
99 
101 
102  if ((cmpindex = _compressframe_findmethod(compression_method)) >= 0) {
103  g_warning("%s.%d: Unknown compression type: %d"
104  , __FUNCTION__, __LINE__, compression_method);
105  return(NULL);
106  }
107  fself = frame_new(frame_type, sizeof(CompressFrame));
108  self = NEWSUBCLASS(CompressFrame, fself);
109  self->compression_index = cmpindex;
110  fself->length = 4;
115 #if 0
116  if (NULL == parentfinalize) {
117  parentfinalize = fself->baseclass._finalize;
118  }
120 #endif
121  return self;
122 }
124 FSTATIC gboolean
125 _compressframe_isvalid(const Frame *fself, gconstpointer tlvstart, gconstpointer pktend)
126 {
127  const CompressFrame* self = CASTTOCONSTCLASS(CompressFrame, fself);
128  guint8 compresstype;
129  guint32 origlen;
130  const guint8* valptr;
131  if (tlvstart == NULL) {
132  return self->compression_index < DIMOF(allcompressions)
133  && allcompressions[self->compression_index].compression_type == self->compression_method;
134  }
135  if ( ((const guint8*)pktend-(const guint8*)tlvstart) < 12
136  || get_generic_tlv_len(tlvstart, pktend) <= 8) {
137  return FALSE;
138  }
139  valptr = get_generic_tlv_value(tlvstart, pktend);
140  compresstype = tlv_get_guint8(valptr, pktend);
141  if (_compressframe_findmethod(compresstype) < 0) {
142  return FALSE;
143  }
144  origlen = tlv_get_guint24(valptr+1, pktend);
145  // Trying to avoid a DOS attack using huge packets
146  if (origlen > MAXUNCOMPRESSEDSIZE || origlen < 32) { // 32 is a guess at min len
147  return FALSE;
148  }
149  return TRUE;
150 }
151 FSTATIC gsize
153 {
154  return 4 + f->length;
155 }
156 
157 FSTATIC void
158 _compressframe_setvalue(Frame *f, gpointer value, guint16 len, GDestroyNotify valnotify)
159 {
160  (void)f;
161  (void)value;
162  (void)len;
163  (void)valnotify;
164  g_warning("%s:%d: Not possible to set the value of a CompressFrame", __FUNCTION__, __LINE__);
165 }
166 
167 #define COMPFRAMESIZE 4
168 
174 FSTATIC void
175 _compressframe_updatedata(Frame *f, gpointer tlvstart, gconstpointer pktend, FrameSet* fs)
176 {
178  guint8* pktstart = fs->packet;
179  guint8* tlvstart8 = tlvstart;
180  const guint8* pktend8 = pktend;
181  guint8* valptr;
182  guint32 offset;
183  gpointer newpacket;
184  int compressedsize;
185 
186  // Write our type and length into the packet
187  set_generic_tlv_type(tlvstart, f->type, pktend);
188  valptr = get_generic_tlv_nonconst_value(tlvstart, pktend);
189  // Our value consists of the compression method followed by a 3 byte
190  // packet length. This restricts us to a 16M decompressed original
191  // packet. That should be big enough for a while...
192  // Of course, the fact that this has to compress down to a single UDP
193  // packet makes this a very reasonable assumption...
194  // In practice, our JSON seems to be limited to about 300K decompressed.
195  tlv_set_guint8(valptr, self->compression_method, pktend);
196  self->decompressed_size = pktend8 - pktstart;
197  tlv_set_guint24(valptr+1, self->decompressed_size, pktend);
198 
199  // Now on to our side effect - compressing the frames that follow us...
200  offset = (tlvstart8+COMPFRAMESIZE)-pktstart;
201  newpacket = allcompressions[self->compression_index].compress
202  (pktstart, self->decompressed_size, offset, MAXUDPSIZE, &compressedsize, 0);
203  if (NULL == newpacket) {
204  g_warning("%s:%d: Unable to compress %d byte packet to %d byte UDP packet"
205  , __FUNCTION__, __LINE__, self->decompressed_size, MAXUDPSIZE);
206  }
207  set_generic_tlv_len(tlvstart, COMPFRAMESIZE+compressedsize, pktend);
208  fs->packet = newpacket;
209 }
210 
211 #define COMPRESSFRAMEMIN 4
212 FSTATIC Frame*
213 compressframe_tlvconstructor(gconstpointer tlvstart,
214  gconstpointer pktend,
215  gpointer* newpacket,
216  gpointer* newpacketend)
217 {
218  const guint8* pktend8 = pktend;
219  const guint8* tlvstart8 = tlvstart;
220  const guint8* packet = tlvstart8 + COMPRESSFRAMEMIN;
221  guint32 pktsize;
222  guint8 compression_type;
223  const guint8* valueptr;
224  guint32 decompressed_size;
225  int actual_size;
226  int compression_index;
227  guint16 frametype;
228  CompressFrame* ret;
229  /* Our four bytes of real data are:
230  * compression type
231  * 3-byte decompressed size
232  */
233  frametype = get_generic_tlv_type(tlvstart, pktend);
234  valueptr = get_generic_tlv_value(tlvstart, pktend);
235  compression_type = tlv_get_guint8(valueptr, pktend);
236  decompressed_size = tlv_get_guint24(valueptr+1, pktend);
237  compression_index = _compressframe_findmethod(compression_type);
238  // Trying to mitigage possible DOS attack using huge packets
239  // In practice, our max JSON decompressed size is under 325K
240  g_return_val_if_fail(decompressed_size <= MAXUNCOMPRESSEDSIZE, NULL);
241  g_return_val_if_fail(decompressed_size > 32, NULL);
242  g_return_val_if_fail(compression_index >= 0, NULL);
243 
244  pktsize = pktend8 - tlvstart8; // Compressed packet size
245 
246  *newpacket = allcompressions[compression_index].decompress
247  (packet, pktsize, 0, decompressed_size, &actual_size);
248  g_return_val_if_fail(newpacket != NULL, NULL);
249  *newpacketend = (guint8*)newpacket + actual_size;
250 
251  ret = compressframe_new(frametype, compression_type);
252  g_return_val_if_fail(ret != NULL, NULL);
253  ret->decompressed_size = decompressed_size;
254  return NULL;
255 }
256 
257 #ifdef HAVE_ZLIB_H
258 gpointer
263 z_compressbuf(gconstpointer inbuf
264 , int insize
265 , int offset
266 , int maxout
267 , int *actualsize
268 , int level) {
269 
270  guint8* outbuf;
271  z_stream stream;
272  int ret;
273 #if 0
274  int space;
275  double ratio;
276 #endif
277 
278  // Compute compression level
279  // If our guess doesn't work, we'll escalate to max compression
280  // This adds compression expense, this is mostly in the nanoprobe side, so we don't much care.
281  if (level < 1) {
282  if (insize < KBYTES(189)) {
283  level = 1;
284  }else if (insize < KBYTES(225)) {
285  level = 6;
286  }else{
287  level = 9;
288  }
289  }
290 
291  /* Set up libz */
292  stream.zalloc = Z_NULL;
293  stream.zfree = Z_NULL;
294  stream.opaque = Z_NULL;
295  ret = deflateInit(&stream, level);
296 
297  if (ret != Z_OK) {
298  g_warning("%s.%d: OOPS, got a deflateInit return of %d"
299  , __FUNCTION__, __LINE__, ret);
300  return NULL;
301  }
302  outbuf = g_malloc(maxout);
303  if (NULL == outbuf) {
304  g_warning("%s.%d: OOPS out of space.", __FUNCTION__, __LINE__);
305  return NULL;
306  }
307  stream.avail_in = insize-offset; stream.next_in = (const guint8*)inbuf+offset;
308  stream.avail_out = maxout; stream.next_out = outbuf+offset;
309 
310  /* Compress it */
311  ret = deflate(&stream, Z_FINISH);
312  if (ret != Z_STREAM_END) {
313  g_free(outbuf);
314  if (level < 9) {
315  return z_compressbuf(inbuf, insize, offset, maxout, actualsize, 9);
316  }
317  g_warning("%s.%d: Got return code %d from deflate."
318  , __FUNCTION__, __LINE__, ret);
319  return NULL;
320  }
321 #if 0
322  space = insize - stream.avail_out;
323  ratio = ((double)stream.total_in/((double)stream.total_out));
324  //return 0;
325  fprintf(stderr, "Compressing %ld bytes into %ld bytes with level %d ratio %.2f:1\n", stream.total_in
326  , stream.total_out, level, ratio);
327 #endif
328  (void)deflateEnd(&stream);
329  outbuf = g_realloc(outbuf, stream.total_out);
330  *actualsize = stream.total_out;
331  return outbuf;
332 }
333 
336 gpointer
337 z_decompressbuf(gconstpointer inbuf
338 , int insize
339 , int offset
340 , int maxout
342 , int *uncompsize) {
344  gpointer outbuf;
345  int outsize;
346  int ret;
347  z_stream stream;
348  /* Set up libz */
349  stream.zalloc = Z_NULL;
350  stream.zfree = Z_NULL;
351  stream.opaque = Z_NULL;
352  stream.avail_in = insize-offset;
353  stream.next_in = ((const guint8*)inbuf) + offset;
354  if (Z_OK != inflateInit(&stream)) {
355  return NULL;
356  }
357  outbuf = g_malloc(maxout);
358  stream.avail_out = maxout;
359  stream.next_out = ((guint8 *)outbuf) + offset;
360  // Decompress our input buffer.
361  ret = inflate(&stream, Z_FINISH);
362  (void)inflateEnd(&stream);
363  if (ret != Z_STREAM_END) {
364  g_warning( "%s.%d: GOT inflate RETURN OF %d"
365  , __FUNCTION__, __LINE__, ret);
366  g_free(outbuf);
367  return NULL;
368  }
369  outsize = maxout - stream.avail_out;
370  if (outsize > maxout) {
371  outbuf = g_realloc(outbuf, outsize);
372  }
373  if (offset > 0) {
374  memcpy(outbuf, inbuf, offset);
375  }
376  (void)inflateEnd(&stream);
377  *uncompsize = outsize;
378  return outbuf;
379 }
380 #endif /* HAVE_ZLIB_H */