9 #include "hkl-binoculars-cnpy-private.h"
10 #include "hkl/ccan/array_size/array_size.h"
11 #include "hkl/ccan/build_assert/build_assert.h"
12 #include "hkl/ccan/darray/darray.h"
14 #define REGEX_DESCR "'descr':\\s+'(.+)'"
15 #define REGEX_FORTRAN "'fortran_order':\\s+(.+)"
16 #define REGEX_SHAPE "'shape':\\s+\\((.+)\\)"
17 #define REGEX_HEADER "^\\{" REGEX_DESCR ",\\s+" REGEX_FORTRAN ",\\s+" REGEX_SHAPE ",\\s+" "\\}\\s+"
25 static const char magic
[] = {(char)0x93, 'N', 'U', 'M', 'P', 'Y'};
27 const uint8_t VERSION_1
= 0x01;
28 const uint8_t VERSION_2
= 0x02;
29 const uint8_t VERSION_3
= 0x03;
39 HklBinocularsNpyDataType elem_type
;
44 struct pre_header_t pre
;
53 void npy_free_but_array(struct npy_t
*npy
)
59 static char *extract_as_string(const char *header
, regmatch_t match
)
61 int size
= match
.rm_eo
- match
.rm_so
;
63 char *buffer
= g_new(char, size
+ 1);
64 strncpy(buffer
, &header
[match
.rm_so
], size
);
70 static struct descr_t
parse_descr(const char *header
, regmatch_t match
)
72 char *description
= extract_as_string(header
, match
);
77 switch(description
[0]){
80 descr
.endianess
= LittleEndian();
85 switch(description
[1]){
87 descr
.elem_type
= HklBinocularsNpyBool();
92 descr
.elem_size
= atoi(&description
[2]);
99 static int parse_fortran_order(const char *header
, regmatch_t match
)
101 int fortran_order
= 0;
103 if(!strncmp(&header
[match
.rm_so
], "True", 4))
106 return fortran_order
;
110 static void parse_shape(const char *header
, regmatch_t match
, darray_int
*shape
)
112 char *repr
= extract_as_string(header
, match
);
122 token
=strtok_r(repr
, ",", &saveptr
);
127 /* TODO deal with atoi errors */
128 darray_append(*shape
, val
);
134 static int shape_cmp(const darray_int
*sh1
, const darray_int
*sh2
)
138 if (darray_size(*sh1
) != darray_size(*sh2
))
139 return -1; /* not idential */
141 for(i
=0; i
<darray_size(*sh1
); ++i
)
142 if(darray_item(*sh1
, i
) != darray_item(*sh2
, i
))
145 return 0; /* idential */
148 static int shape_size(const darray_int
*shape
)
153 darray_foreach(val
, *shape
){
160 static struct npy_t
*parse_npy(FILE* fp
,
161 HklBinocularsNpyDataType type
,
162 const darray_int
*shape
)
164 struct npy_t
*npy
= g_new0(struct npy_t
, 1);
167 regmatch_t matches
[4];
171 BUILD_ASSERT(sizeof(npy
->pre
) == 8);
173 /* read the pre_header */
174 res
= fread(&npy
->pre
, 1, sizeof(npy
->pre
), fp
);
177 if(strncmp(magic
, npy
->pre
.magic
, ARRAY_SIZE(magic
)))
180 /* read the header size */
181 switch (npy
->pre
.major
) {
185 res
= fread(&len
, 1, sizeof(len
), fp
);
187 npy
->header_len
= (uint32_t)len
;
192 res
= fread(&npy
->header_len
, 1, sizeof(npy
->header_len
), fp
);
198 res
= fread(&npy
->header_len
, 1, sizeof(npy
->header_len
), fp
);
204 /* read the header */
206 npy
->header
= malloc(npy
->header_len
* sizeof(char));
207 res
= fread(npy
->header
, 1, npy
->header_len
, fp
);
208 assert(res
== npy
->header_len
);
211 /* parse the header */
212 /* {descr: '|b1', fortran_order: False, shape: (240, 560), } */
214 errcode
= regcomp(&preg
, REGEX_HEADER
, REG_EXTENDED
);
216 regerror(errcode
, &preg
, errbuf
, 256);
217 fprintf(stdout
, "errcode: %s\n", errbuf
);
222 errcode
= regexec(&preg
, npy
->header
, ARRAY_SIZE(matches
), matches
, 0);
224 regerror(errcode
, &preg
, errbuf
, 256);
225 fprintf(stdout
, "errcode: %s\n", errbuf
);
233 npy
->descr
= parse_descr(npy
->header
, matches
[1]);
234 npy
->fortran_order
= parse_fortran_order(npy
->header
, matches
[2]);
235 parse_shape(npy
->header
, matches
[3], &npy
->shape
);
237 if(type
.tag
!= npy
->descr
.elem_type
.tag
)
240 if(0 != shape_cmp(shape
, &npy
->shape
))
244 int nbytes
= shape_size(&npy
->shape
) * npy
->descr
.elem_size
;
245 npy
->arr
= malloc( nbytes
);
249 res
= fread(npy
->arr
, 1, nbytes
, fp
);
250 assert(res
== nbytes
);
261 void *npy_load(const char *fname
,
262 HklBinocularsNpyDataType type
,
263 const darray_int
*shape
)
266 FILE* fp
= fopen(fname
, "rb");
269 struct npy_t
*npy
= parse_npy(fp
, type
, shape
);
272 npy_free_but_array(npy
);
280 static inline char bigendian(void)
283 return (((char *)&x
)[0]) ? '<' : '>';
286 static inline char map_type(HklBinocularsNpyDataType type
)
291 of(HklBinocularsNpyBool
) res
= 'b';
292 of(HklBinocularsNpyDouble
) res
= 'f';
295 /* if(t == typeid(float) ) return 'f'; */
296 /* if(t == typeid(double) ) return 'f'; */
297 /* if(t == typeid(long double) ) return 'f'; */
299 /* if(t == typeid(int) ) return 'i'; */
300 /* if(t == typeid(char) ) return 'i'; */
301 /* if(t == typeid(short) ) return 'i'; */
302 /* if(t == typeid(long) ) return 'i'; */
303 /* if(t == typeid(long long) ) return 'i'; */
305 /* if(t == typeid(unsigned char) ) return 'u'; */
306 /* if(t == typeid(unsigned short) ) return 'u'; */
307 /* if(t == typeid(unsigned long) ) return 'u'; */
308 /* if(t == typeid(unsigned long long) ) return 'u'; */
309 /* if(t == typeid(unsigned int) ) return 'u'; */
311 /* if(t == typeid(bool) ) return 'b'; */
313 /* if(t == typeid(std::complex<float>) ) return 'c'; */
314 /* if(t == typeid(std::complex<double>) ) return 'c'; */
315 /* if(t == typeid(std::complex<long double>) ) return 'c'; */
317 /* else return '?'; */
321 static inline int map_size(HklBinocularsNpyDataType type
)
326 of(HklBinocularsNpyBool
) res
= 1;
327 of(HklBinocularsNpyDouble
) res
= 8;
333 static inline char *create_npy_header(HklBinocularsNpyDataType type
,
334 const darray_int
*shape
,
343 /* first compute the dict and its size */
345 stream
= open_memstream(&dict
, &dict_size
);
346 if (NULL
== stream
) goto fail
;
349 fprintf(stream
, "'descr': '%c%c%d'",
350 bigendian(), map_type(type
), map_size(type
));
351 fprintf(stream
, ", ");
352 fprintf(stream
, "'fortran_order' : False");
353 fprintf(stream
, ", ");
354 fprintf(stream
, "'shape': (");
355 fprintf(stream
, "%d", darray_item(*shape
, 0));
356 for(i
=1; i
<darray_size(*shape
); ++i
){
357 fprintf(stream
, ", %d", darray_item(*shape
, i
));
359 fprintf(stream
, ")");
360 fprintf(stream
, ",");
361 fprintf(stream
, "}");
365 /* ``len(magic string) + 2 + len(length) + HEADER_LEN`` be
366 evenly divisible by 64 for alignment purposes. */
367 size_t extra
= 64 - (ARRAY_SIZE(magic
) + 2 + 2 + dict_size
) % 64;
368 for(i
=0; i
<extra
- 1; ++i
){
369 fprintf(stream
, " ");
371 fprintf(stream
, "\n");
375 /* create the full header */
377 stream
= open_memstream(&header
, header_size
);
378 if (NULL
== stream
) goto fail
;
380 fwrite(&magic
[0], sizeof(char), ARRAY_SIZE(magic
), stream
);
381 fprintf(stream
, "%c%c", VERSION_1
, 0x0);
382 uint16_t len
= (uint16_t)dict_size
;
383 fwrite(&len
, 1, sizeof(len
), stream
);
384 fwrite(dict
, 1, dict_size
, stream
);
385 /* fprintf(stdout, "%s", dict); */
396 void npy_save(const char *fname
,
398 HklBinocularsNpyDataType type
,
399 const darray_int
*shape
)
405 header
= create_npy_header(type
, shape
, &header_size
);
406 if (NULL
== header
) return;
408 f
= fopen(fname
, "wb");
409 if (NULL
== f
) return;
412 fwrite(&header
[0], 1, header_size
, f
);
413 /* fprintf(stdout, "%s\n", header); */
415 fwrite(arr
, map_size(type
), shape_size(shape
), f
);