2 * =====================================================================================
6 * Description: Floating point compression/Decompression algorithm
9 * Created: 09/02/2012 02:53:08 PM
16 * =====================================================================================
29 #include <smmintrin.h>
34 struct timeval tv
; int ok
;
35 ok
= gettimeofday(&tv
, NULL
);
36 if (ok
<0) { CmiPrintf("gettimeofday error"); }
37 return (tv
.tv_sec
* 1.0 + tv
.tv_usec
* 1.0E-6);
43 #define FLOAT_BIT CHAR_BIT*sizeof(float)
44 #define FLOAT_BYTE sizeof(float)
46 #define COMPRESS_EXP 1
49 #define SETBIT(dest, i) (dest[i>>3]) |= (1 << (i&7) )
50 #define TESTBIT(dest, i) ((dest[i>>3]) >> (i&7)) & 1
51 #define SETBIT11(dest, i) (dest[(i)>>3]) |= (3 << ((i)&7) )
52 #define TESTBIT11(dest, i) ((dest[(i)>>3]) >> ((i)&7)) & 0x3l
56 #define TESTBIT(data, b) (data>>(b)) & 1
57 #define SETBIT(data, index, bit) (data |= ((bit)<<(index)))
60 /** compress char is the general algorithm for any data*/
61 void compressChar(void *src
, void *dst
, int size
, int *compressSize
, void *bData
)
63 register char *source
= (char*)src
;
64 register char *dest
= (char*)dst
;
65 register char *baseData
= (char*)bData
;
68 double t1
= get_clock();
72 memcpy(dest
, source
, size
*sizeof(char));
75 register int _dataIndex
= (size
+7)/8;
76 memset(dest
, 0, (size
+7)/8 );
77 for (i
= 0; i
< size
&&_dataIndex
<size
; ++i
) {
78 // Bitmask everything but the exponents, then check if they match.
79 char xor_d
= source
[i
] ^ baseData
[i
];
80 short different
= xor_d
& 0xff;
82 // If not, mark this exponent as "different" and store it to send with the message.
83 dest
[_dataIndex
] = source
[i
];
91 *compressSize
= _dataIndex
;
94 double t
= get_clock()-t1
;
95 printf(" +++++CHAR done compressing(%d===>%d) (reduction:%d) ration=%f time=%d us\n", (int)(size
*sizeof(char)), *compressSize
, (int)(size
*sizeof(char)-*compressSize
), (1-(float)*compressSize
/(size
*sizeof(char)))*100, (int)(t
*1000000));
99 void decompressChar(void *cData
, void *dData
, int size
, int compressSize
, void *bData
) {
101 double t1
= get_clock();
104 memcpy(dData
, cData
, size
*sizeof(char));
106 char *compressData
= (char*)cData
;
107 char *baseData
= (char*)bData
;
108 register char *decompressData
=(char*)dData
;
109 register int sdataIndex
= (size
+7)/8;
110 register char *src
= (char*)compressData
;
112 for(i
=0; i
<size
; ++i
)
114 if(TESTBIT(src
, i
)) // same
116 decompressData
[i
] = baseData
[i
];
117 }else //different exponet
119 decompressData
[i
] = compressData
[sdataIndex
];
125 double t
= get_clock()-t1
;
126 printf("------CHAR done decompressing..... orig size:%d time:%d us \n", (int)size
, (int)(t
*1000000)) ;
134 void compressFloatingPoint(void *src
, void *dst
, int s
, int *compressSize
, void *bData
)
136 int size
= s
/FLOAT_BYTE
;
137 float *source
= (float*)src
;
138 float *dest
= (float*)dst
;
139 float *baseData
= (float*)bData
;
140 register unsigned int *bptr
= (unsigned int*) baseData
;
141 register unsigned int *uptr
= (unsigned int *) source
;
142 register char *uchar
;
145 double t1
= get_clock();
149 memcpy(dest
, source
, size
*sizeof(float));
152 assert(baseData
!= NULL
);
154 // Create message to receive the compressed buffer.
155 register unsigned char *cdst
= (unsigned char*)dest
;
156 register int _dataIndex
= (size
+7)/8;
157 register unsigned int diff
;
158 memset(cdst
, 0, (size
+7)/8 );
160 register const __m128i
* b_ptr
= (__m128i
*)bptr
;
161 register const __m128i
* u_ptr
= (__m128i
*)uptr
;
163 register __m128i xmm_f
= _mm_set1_epi32(0xFF000000);
164 for (i
= 0; i
< size
; i
+=4) {
165 // Bitmask everything but the exponents, then check if they match.
166 __m128i xmm_b
= _mm_load_si128(b_ptr
);
167 __m128i xmm_u
= _mm_load_si128(u_ptr
);
168 __m128i xmm_d
= _mm_xor_si128(xmm_b
, xmm_u
); // XOR 4 32-bit words
169 xmm_d
= _mm_and_si128(xmm_d
, xmm_f
);
171 if (_mm_extract_epi32(xmm_d
, 0)) {
173 memcpy(cdst
+_dataIndex
, &(uptr
[i
]), 4);
177 memcpy(cdst
+_dataIndex
, &(uptr
[i
]), 3);
180 if (_mm_extract_epi32(xmm_d
, 1)) {
182 memcpy(cdst
+_dataIndex
, &(uptr
[i
+1]), 4);
185 memcpy(cdst
+_dataIndex
, &(uptr
[i
+1]), 3);
188 if (_mm_extract_epi32(xmm_d
, 2)) {
190 memcpy(cdst
+_dataIndex
, &(uptr
[i
+2]), 4);
193 memcpy(cdst
+_dataIndex
, &(uptr
[i
+2]), 3);
196 if (_mm_extract_epi32(xmm_d
, 3)) {
198 memcpy(cdst
+_dataIndex
, &(uptr
[i
+3]), 4);
201 memcpy(cdst
+_dataIndex
, &(uptr
[i
+3]), 3);
207 *compressSize
= _dataIndex
;
211 double t
= get_clock()-t1
;
212 printf(" ===>floating compare done compressingcompressed size:(%d===>%d) (reduction:%d) ration=%f time=%d us \n", (int)(size
*sizeof(float)), *compressSize
, (int)(size
*sizeof(float)-*compressSize
), (1-(float)*compressSize
/(size
*sizeof(float)))*100, (int)(t
*1000000));
218 void compressFloatingPoint(void *src
, void *dst
, int s
, int *compressSize
, void *bData
)
220 int size
= s
/FLOAT_BYTE
;
221 float *source
= (float*)src
;
222 float *dest
= (float*)dst
;
223 float *baseData
= (float*)bData
;
224 register unsigned int *bptr
= (unsigned int*) baseData
;
225 register unsigned int *uptr
= (unsigned int *) source
;
226 register char *uchar
;
229 double t1
= get_clock();
233 memcpy(dest
, source
, size
*sizeof(float));
236 assert(baseData
!= NULL
);
237 // Is this the first time we're sending stuff to this node?
239 // Create message to receive the compressed buffer.
240 register unsigned char *cdst
= (unsigned char*)dest
;
241 register int _dataIndex
= (size
+7)/8;
242 register unsigned int diff
;
243 memset(cdst
, 0, (size
+7)/8 );
244 for (i
= 0; i
< size
; ++i
) {
245 // Bitmask everything but the exponents, then check if they match.
246 diff
= (bptr
[i
] ^ uptr
[i
]) & 0xff000000 ;
248 // If not, mark this exponent as "different" and store it to send with the message.
250 memcpy(cdst
+_dataIndex
, &(uptr
[i
]), 4);
254 memcpy(cdst
+_dataIndex
, &(uptr
[i
]), 3);
259 *compressSize
= _dataIndex
;
263 double t
= get_clock()-t1
;
264 CmiPrintf(" ===> FLOATING done compressingcompressed size:(%d===>%d) (reduction:%d) ration=%f time=%d us\n", (int)(size
*sizeof(float)), *compressSize
, (int)(size
*sizeof(float)-*compressSize
), (1-(float)*compressSize
/(size
*sizeof(float)))*100, (int)(t
*1000000));
270 void decompressFloatingPoint(void *cData
, void *dData
, int s
, int compressSize
, void *bData
) {
271 int size
= s
/FLOAT_BYTE
;
273 double t1
= get_clock();
276 memcpy(dData
, cData
, size
*sizeof(float));
278 float *compressData
= (float*)cData
;
279 float *baseData
= (float*)bData
;
280 register unsigned int *decompressData
=(unsigned int*)dData
;
281 register int _sdataIndex
= (size
+7)/8;
282 register char *src
= (char*)compressData
;
283 register int exponent
;
284 register unsigned int mantissa
;
285 register unsigned int *bptr
= (unsigned int*)baseData
;
287 for(i
=0; i
<size
; ++i
)
289 if(TESTBIT(src
, i
)) // different
292 decompressData
[i
] = *((unsigned int*)(src
+_sdataIndex
));
296 exponent
= bptr
[i
] & 0xff000000;
297 mantissa
= *((unsigned int*)(src
+_sdataIndex
)) & 0x00FFFFFF;
298 mantissa
|= exponent
;
299 decompressData
[i
] = mantissa
;
305 double t
= get_clock()-t1
;
306 //CmiPrintf("--- FLOATING done decompressing..... orig size:%d\n time:%d us", (int)size, (int)(t*1000000)) ;
312 void compressFloatingPoint(void *src
, void *dst
, int s
, int *compressSize
, void *bData
)
314 register unsigned int *dest
= (unsigned int*)dst
;
315 register unsigned int *bptr
= (unsigned int*) bData
;
316 register unsigned int *uptr
= (unsigned int *) src
;
317 int size
= s
/sizeof(float);
319 double t1
= get_clock();
323 memcpy(dest
, src
, size
*sizeof(float));
326 register unsigned int comp_data
= 0;
327 register int f_index
= 0;
332 register unsigned int xor_data
;
334 for (i
= 0; i
< size
; ++i
) {
335 xor_data
= (uptr
[i
])^(bptr
[i
]);
338 while(!TESTBIT(xor_data
, b
) && zers
<15){
345 SETBIT(dest
[(int)(f_index
>>5)], (f_index
&0x1f), TESTBIT(zers
, j
));
350 SETBIT(dest
[(f_index
>>5)], f_index
&0x1f, TESTBIT(xor_data
, b
));
355 *compressSize
= f_index
/8;
356 float compressRatio
= (1-(float)(*compressSize
)/s
)*100;
359 double t
= get_clock()-t1
;
360 CmiPrintf("===>[floating point lzc]done compressing compressed size:(%d===>%d) (reduction:%d) ration=%f Timer:%d us\n\n", (int)(size
*sizeof(float)), *compressSize
, (int)((size
*sizeof(float)-*compressSize
)), (1-(float)*compressSize
/(size
*sizeof(float)))*100, (int)(t
*1000000));
366 void decompressFloatingPoint(void *cData
, void *dData
, int s
, int compressSize
, void *bData
) {
367 int size
= s
/sizeof(float);
369 double t1
= get_clock();
371 CmiPrintf("[%d] starting decompressing \n", CmiMyPe());
374 memcpy(dData
, cData
, size
*sizeof(float));
376 register unsigned int *compressData
= (unsigned int*)cData
;
377 register unsigned int *decompressData
= (unsigned int*)dData
;
378 register unsigned int *baseData
= (unsigned int*)bData
;
379 memset(decompressData
, 0, s
);
381 register unsigned int xor_data
;
382 register int data
= 0;
383 register int d_index
=0;
384 register int compp
= 0;
388 for (i
=0; i
<size
; i
++) {
391 //read 4 bits and puts index acccordingly
392 for (f
=0; f
<4; f
++,compp
++) {
393 if(TESTBIT(compressData
[(int)(compp
>>5)], (compp
&0x1f))){
394 for (j
=0; j
< (1<<f
); j
++) {
395 SETBIT(data
, index
, 0);
401 SETBIT(data
, index
, TESTBIT(compressData
[(int)(compp
>>5)], (compp
&0x1f)));
404 xor_data
= data
^(baseData
[i
]);
405 decompressData
[i
] = xor_data
;
409 double t
= get_clock()-t1
;
411 CmiPrintf("[%d] done decompressing..... orig size:%d time:%d us \n", CmiMyPe(), size
, (int)(t
*1000000));
420 /***************************
422 * algorithms to compress doubles
425 #define DOUBLE_BYTE sizeof(double)
426 #define BITS_DOUBLE sizeof(double)*8
430 void compressDouble(void *src
, void *dst
, int s
, int *compressSize
, void *bData
)
432 int size
= s
/DOUBLE_BYTE
;
433 double *source
= (double*)src
;
434 double *dest
= (double*)dst
;
435 double *baseData
= (double*)bData
;
436 register unsigned long *bptr
= (unsigned long*) baseData
;
437 register unsigned long *uptr
= (unsigned long*) source
;
438 register char *uchar
;
441 double t1
= get_clock();
445 memcpy(dest
, source
, s
);
448 assert(baseData
!= NULL
);
449 // Is this the first time we're sending stuff to this node?
452 // Create message to receive the compressed buffer.
453 register unsigned char *cdst
= (unsigned char*)dest
;
454 register int _dataIndex
= (2*size
+7)/8;
455 memset(cdst
, 0, (2*size
+7)/8 );
456 for (i
= 0; i
< size
; ++i
) {
457 // Bitmask everything but the exponents, then check if they match.
458 unsigned long xord
= bptr
[i
] ^ uptr
[i
];
459 unsigned long eight
= xord
& 0xff00000000000000;
460 unsigned long sixteen
= xord
& 0xffff000000000000;
461 if(sixteen
== 0l) //00
463 unsigned long ui
= uptr
[i
];
464 memcpy(cdst
+_dataIndex
, &ui
, 6);
467 else if(eight
== 0l)//01
470 unsigned long ui
= uptr
[i
];
471 memcpy(cdst
+_dataIndex
, &ui
, 7);
475 SETBIT11(cdst
, i
<<1);
476 unsigned long ui
= uptr
[i
];
477 memcpy(cdst
+_dataIndex
, &ui
, 8);
481 *compressSize
= _dataIndex
;
485 double t
= get_clock()-t1
;
486 printf(" ===>[double lzc] done compressingcompressed size:(%d===>%d) (reduction:%d) ration=%f time=%d us\n", (int)(size
*sizeof(double)), *compressSize
, (int)(size
*sizeof(double)-*compressSize
), (1-(double)*compressSize
/(size
*sizeof(double)))*100, (int)(t
*1000000));
490 void decompressDouble(void *cData
, void *dData
, int s
, int compressSize
, void *bData
) {
491 int size
= s
/DOUBLE_BYTE
;
493 double t1
= get_clock();
496 memcpy(dData
, cData
, s
);
498 double *compressData
= (double*)cData
;
499 double *baseData
= (double*)bData
;
500 register unsigned long *decompressData
=(unsigned long*)dData
;
501 register int _sdataIndex
= (2*size
+7)/8;
502 register char *src
= (char*)compressData
;
503 register unsigned long exponent
;
504 register unsigned long mantissa
;
505 register unsigned long *bptr
= (unsigned long*)baseData
;
507 for(i
=0; i
<size
; ++i
)
509 int bitss
= TESTBIT(src
, i
<<1);
510 if(bitss
==3) // different
513 decompressData
[i
] = *((unsigned long*)(src
+_sdataIndex
));
517 exponent
= bptr
[i
] & 0xff00000000000000;
518 mantissa
= *((unsigned long*)(src
+_sdataIndex
)) & 0x00ffffffffffffff;
519 mantissa
|= exponent
;
520 decompressData
[i
] = mantissa
;
524 exponent
= bptr
[i
] & 0xffff000000000000;
525 mantissa
= *((unsigned long*)(src
+_sdataIndex
)) & 0x0000ffffffffffff;
526 mantissa
|= exponent
;
527 decompressData
[i
] = mantissa
;
533 double t
= get_clock()-t1
;
534 printf("done decompressing..... orig size:%d\n time:%d us", (int)size
, (int)(t
*1000000)) ;
542 void compressDouble(void *src
, void *dst
, int s
, int *compressSize
, void *bData
)
544 register unsigned long *dest
= (unsigned long*)dst
;
545 register unsigned long *bptr
= (unsigned long*) bData
;
546 register unsigned long *uptr
= (unsigned long*) src
;
547 int size
= s
/sizeof(double);
549 double t1
= get_clock();
553 memcpy(dest
, src
, size
*sizeof(double));
556 register int f_index
= 0;
561 register unsigned long xor_data
;
563 for (i
= 0; i
< size
; ++i
) {
564 xor_data
= (uptr
[i
])^(bptr
[i
]);
566 //int value = xor_data;
567 //printbitssimple(value);
570 while(!TESTBIT(xor_data
, b
) && zers
<15){
574 //cout<<"c: "<<zers<<endl;
578 SETBIT(dest
[(int)(f_index
>>6)], (f_index
&0x3f), ((unsigned long)(TESTBIT(zers
, j
))));
583 SETBIT(dest
[(f_index
>>6)], f_index
&0x3f, TESTBIT(xor_data
, b
));
588 /*for (int k=0; k<size; k++) {
589 printf(" %f ",dest[k]);
592 *compressSize
= f_index
/8;
593 double compressRatio
= (1-(double)(*compressSize
)/s
)*100;
596 double t
= get_clock()-t1
;
597 printf("===>double lzc done compressing compressed size:(%d===>%d) (reduction:%d) ration=%f Timer:%d us\n\n", (int)(size
*sizeof(double)), *compressSize
, (int)((size
*sizeof(double)-*compressSize
)), (1-(double)*compressSize
/(size
*sizeof(double)))*100, (int)(t
*1000000));
603 void decompressDouble(void *cData
, void *dData
, int s
, int compressSize
, void *bData
) {
604 int size
= s
/sizeof(double);
606 double t1
= get_clock();
609 memcpy(dData
, cData
, size
*sizeof(double));
611 register unsigned long *compressData
= (unsigned long*)cData
;
612 register unsigned long *decompressData
= (unsigned long*)dData
;
613 register unsigned long *baseData
= (unsigned long*)bData
;
614 /*for (int k=0; k<size; k++) {
615 printf("d: %d ",compressData[k]);
618 memset(decompressData
, 0, s
);
620 register unsigned long xor_data
;
621 register unsigned long data
= 0;
622 register int d_index
=0;
623 register int compp
= 0;
627 for (i
=0; i
<size
; i
++) {
628 index
= BITS_DOUBLE
-1;
629 data
= 0; int zers
=0;
630 //read 4 bits and puts index acccordingly
631 for (f
=0; f
<4; f
++,compp
++) {
632 if(TESTBIT(compressData
[(int)(compp
>>6)], (compp
&0x3f))){
633 for (j
=0; j
< (1<<f
); j
++) {
638 //cout<<"d: "<<zers<<endl;
641 SETBIT(data
, index
, TESTBIT(compressData
[(int)(compp
>>6)], (compp
&0x3f)));
644 xor_data
= data
^(baseData
[i
]);
645 decompressData
[i
] = xor_data
;
649 double t
= get_clock()-t1
;
650 printf("done decompressing..... orig size:%d time:%d us \n", size
, (int)(t
*1000000));