2 /* filter_neon_intrinsics.c - NEON optimised filter functions
4 * Copyright (c) 2014 Glenn Randers-Pehrson
5 * Written by James Yu <james.yu at linaro.org>, October 2013.
6 * Based on filter_neon.S, written by Mans Rullgard, 2011.
8 * Last changed in libpng 1.6.16 [December 22, 2014]
10 * This code is released under the libpng license.
11 * For conditions of distribution and use, see the disclaimer
12 * and license in png.h
15 #include "../pngpriv.h"
17 #ifdef PNG_READ_SUPPORTED
19 /* This code requires -mfpu=neon on the command line: */
20 #if PNG_ARM_NEON_IMPLEMENTATION == 1 /* intrinsics code from pngpriv.h */
24 /* libpng row pointers are not necessarily aligned to any particular boundary,
25 * however this code will only work with appropriate alignment. arm/arm_init.c
26 * checks for this (and will not compile unless it is done). This code uses
27 * variants of png_aligncast to avoid compiler warnings.
29 #define png_ptr(type,pointer) png_aligncast(type *,pointer)
30 #define png_ptrc(type,pointer) png_aligncastconst(const type *,pointer)
32 /* The following relies on a variable 'temp_pointer' being declared with type
33 * 'type'. This is written this way just to hide the GCC strict aliasing
34 * warning; note that the code is safe because there never is an alias between
35 * the input and output pointers.
37 #define png_ldr(type,pointer)\
38 (temp_pointer = png_ptr(type,pointer), *temp_pointer)
40 #if PNG_ARM_NEON_OPT > 0
43 png_read_filter_row_up_neon(png_row_infop row_info
, png_bytep row
,
44 png_const_bytep prev_row
)
47 png_bytep rp_stop
= row
+ row_info
->rowbytes
;
48 png_const_bytep pp
= prev_row
;
50 for (; rp
< rp_stop
; rp
+= 16, pp
+= 16)
56 qrp
= vaddq_u8(qrp
, qpp
);
62 png_read_filter_row_sub3_neon(png_row_infop row_info
, png_bytep row
,
63 png_const_bytep prev_row
)
66 png_bytep rp_stop
= row
+ row_info
->rowbytes
;
68 uint8x16_t vtmp
= vld1q_u8(rp
);
69 uint8x8x2_t
*vrpt
= png_ptr(uint8x8x2_t
, &vtmp
);
70 uint8x8x2_t vrp
= *vrpt
;
73 vdest
.val
[3] = vdup_n_u8(0);
77 uint8x8_t vtmp1
, vtmp2
;
78 uint32x2_t
*temp_pointer
;
80 vtmp1
= vext_u8(vrp
.val
[0], vrp
.val
[1], 3);
81 vdest
.val
[0] = vadd_u8(vdest
.val
[3], vrp
.val
[0]);
82 vtmp2
= vext_u8(vrp
.val
[0], vrp
.val
[1], 6);
83 vdest
.val
[1] = vadd_u8(vdest
.val
[0], vtmp1
);
85 vtmp1
= vext_u8(vrp
.val
[1], vrp
.val
[1], 1);
86 vdest
.val
[2] = vadd_u8(vdest
.val
[1], vtmp2
);
87 vdest
.val
[3] = vadd_u8(vdest
.val
[2], vtmp1
);
89 vtmp
= vld1q_u8(rp
+ 12);
90 vrpt
= png_ptr(uint8x8x2_t
, &vtmp
);
93 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[0]), 0);
95 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[1]), 0);
97 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[2]), 0);
99 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[3]), 0);
107 png_read_filter_row_sub4_neon(png_row_infop row_info
, png_bytep row
,
108 png_const_bytep prev_row
)
111 png_bytep rp_stop
= row
+ row_info
->rowbytes
;
114 vdest
.val
[3] = vdup_n_u8(0);
116 for (; rp
< rp_stop
; rp
+= 16)
118 uint32x2x4_t vtmp
= vld4_u32(png_ptr(uint32_t,rp
));
119 uint8x8x4_t
*vrpt
= png_ptr(uint8x8x4_t
,&vtmp
);
120 uint8x8x4_t vrp
= *vrpt
;
121 uint32x2x4_t
*temp_pointer
;
123 vdest
.val
[0] = vadd_u8(vdest
.val
[3], vrp
.val
[0]);
124 vdest
.val
[1] = vadd_u8(vdest
.val
[0], vrp
.val
[1]);
125 vdest
.val
[2] = vadd_u8(vdest
.val
[1], vrp
.val
[2]);
126 vdest
.val
[3] = vadd_u8(vdest
.val
[2], vrp
.val
[3]);
127 vst4_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2x4_t
,&vdest
), 0);
134 png_read_filter_row_avg3_neon(png_row_infop row_info
, png_bytep row
,
135 png_const_bytep prev_row
)
138 png_const_bytep pp
= prev_row
;
139 png_bytep rp_stop
= row
+ row_info
->rowbytes
;
145 vdest
.val
[3] = vdup_n_u8(0);
148 vrpt
= png_ptr(uint8x8x2_t
,&vtmp
);
151 for (; rp
< rp_stop
; pp
+= 12)
153 uint8x8_t vtmp1
, vtmp2
, vtmp3
;
158 uint32x2_t
*temp_pointer
;
161 vppt
= png_ptr(uint8x8x2_t
,&vtmp
);
164 vtmp1
= vext_u8(vrp
.val
[0], vrp
.val
[1], 3);
165 vdest
.val
[0] = vhadd_u8(vdest
.val
[3], vpp
.val
[0]);
166 vdest
.val
[0] = vadd_u8(vdest
.val
[0], vrp
.val
[0]);
168 vtmp2
= vext_u8(vpp
.val
[0], vpp
.val
[1], 3);
169 vtmp3
= vext_u8(vrp
.val
[0], vrp
.val
[1], 6);
170 vdest
.val
[1] = vhadd_u8(vdest
.val
[0], vtmp2
);
171 vdest
.val
[1] = vadd_u8(vdest
.val
[1], vtmp1
);
173 vtmp2
= vext_u8(vpp
.val
[0], vpp
.val
[1], 6);
174 vtmp1
= vext_u8(vrp
.val
[1], vrp
.val
[1], 1);
176 vtmp
= vld1q_u8(rp
+ 12);
177 vrpt
= png_ptr(uint8x8x2_t
,&vtmp
);
180 vdest
.val
[2] = vhadd_u8(vdest
.val
[1], vtmp2
);
181 vdest
.val
[2] = vadd_u8(vdest
.val
[2], vtmp3
);
183 vtmp2
= vext_u8(vpp
.val
[1], vpp
.val
[1], 1);
185 vdest
.val
[3] = vhadd_u8(vdest
.val
[2], vtmp2
);
186 vdest
.val
[3] = vadd_u8(vdest
.val
[3], vtmp1
);
188 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[0]), 0);
190 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[1]), 0);
192 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[2]), 0);
194 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[3]), 0);
200 png_read_filter_row_avg4_neon(png_row_infop row_info
, png_bytep row
,
201 png_const_bytep prev_row
)
204 png_bytep rp_stop
= row
+ row_info
->rowbytes
;
205 png_const_bytep pp
= prev_row
;
208 vdest
.val
[3] = vdup_n_u8(0);
210 for (; rp
< rp_stop
; rp
+= 16, pp
+= 16)
213 uint8x8x4_t
*vrpt
, *vppt
;
214 uint8x8x4_t vrp
, vpp
;
215 uint32x2x4_t
*temp_pointer
;
217 vtmp
= vld4_u32(png_ptr(uint32_t,rp
));
218 vrpt
= png_ptr(uint8x8x4_t
,&vtmp
);
220 vtmp
= vld4_u32(png_ptrc(uint32_t,pp
));
221 vppt
= png_ptr(uint8x8x4_t
,&vtmp
);
224 vdest
.val
[0] = vhadd_u8(vdest
.val
[3], vpp
.val
[0]);
225 vdest
.val
[0] = vadd_u8(vdest
.val
[0], vrp
.val
[0]);
226 vdest
.val
[1] = vhadd_u8(vdest
.val
[0], vpp
.val
[1]);
227 vdest
.val
[1] = vadd_u8(vdest
.val
[1], vrp
.val
[1]);
228 vdest
.val
[2] = vhadd_u8(vdest
.val
[1], vpp
.val
[2]);
229 vdest
.val
[2] = vadd_u8(vdest
.val
[2], vrp
.val
[2]);
230 vdest
.val
[3] = vhadd_u8(vdest
.val
[2], vpp
.val
[3]);
231 vdest
.val
[3] = vadd_u8(vdest
.val
[3], vrp
.val
[3]);
233 vst4_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2x4_t
,&vdest
), 0);
238 paeth(uint8x8_t a
, uint8x8_t b
, uint8x8_t c
)
241 uint16x8_t p1
, pa
, pb
, pc
;
243 p1
= vaddl_u8(a
, b
); /* a + b */
244 pc
= vaddl_u8(c
, c
); /* c * 2 */
245 pa
= vabdl_u8(b
, c
); /* pa */
246 pb
= vabdl_u8(a
, c
); /* pb */
247 pc
= vabdq_u16(p1
, pc
); /* pc */
249 p1
= vcleq_u16(pa
, pb
); /* pa <= pb */
250 pa
= vcleq_u16(pa
, pc
); /* pa <= pc */
251 pb
= vcleq_u16(pb
, pc
); /* pb <= pc */
253 p1
= vandq_u16(p1
, pa
); /* pa <= pb && pa <= pc */
258 d
= vbsl_u8(d
, b
, c
);
259 e
= vbsl_u8(e
, a
, d
);
265 png_read_filter_row_paeth3_neon(png_row_infop row_info
, png_bytep row
,
266 png_const_bytep prev_row
)
269 png_const_bytep pp
= prev_row
;
270 png_bytep rp_stop
= row
+ row_info
->rowbytes
;
275 uint8x8_t vlast
= vdup_n_u8(0);
277 vdest
.val
[3] = vdup_n_u8(0);
280 vrpt
= png_ptr(uint8x8x2_t
,&vtmp
);
283 for (; rp
< rp_stop
; pp
+= 12)
287 uint8x8_t vtmp1
, vtmp2
, vtmp3
;
288 uint32x2_t
*temp_pointer
;
291 vppt
= png_ptr(uint8x8x2_t
,&vtmp
);
294 vdest
.val
[0] = paeth(vdest
.val
[3], vpp
.val
[0], vlast
);
295 vdest
.val
[0] = vadd_u8(vdest
.val
[0], vrp
.val
[0]);
297 vtmp1
= vext_u8(vrp
.val
[0], vrp
.val
[1], 3);
298 vtmp2
= vext_u8(vpp
.val
[0], vpp
.val
[1], 3);
299 vdest
.val
[1] = paeth(vdest
.val
[0], vtmp2
, vpp
.val
[0]);
300 vdest
.val
[1] = vadd_u8(vdest
.val
[1], vtmp1
);
302 vtmp1
= vext_u8(vrp
.val
[0], vrp
.val
[1], 6);
303 vtmp3
= vext_u8(vpp
.val
[0], vpp
.val
[1], 6);
304 vdest
.val
[2] = paeth(vdest
.val
[1], vtmp3
, vtmp2
);
305 vdest
.val
[2] = vadd_u8(vdest
.val
[2], vtmp1
);
307 vtmp1
= vext_u8(vrp
.val
[1], vrp
.val
[1], 1);
308 vtmp2
= vext_u8(vpp
.val
[1], vpp
.val
[1], 1);
310 vtmp
= vld1q_u8(rp
+ 12);
311 vrpt
= png_ptr(uint8x8x2_t
,&vtmp
);
314 vdest
.val
[3] = paeth(vdest
.val
[2], vtmp2
, vtmp3
);
315 vdest
.val
[3] = vadd_u8(vdest
.val
[3], vtmp1
);
319 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[0]), 0);
321 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[1]), 0);
323 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[2]), 0);
325 vst1_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2_t
,&vdest
.val
[3]), 0);
331 png_read_filter_row_paeth4_neon(png_row_infop row_info
, png_bytep row
,
332 png_const_bytep prev_row
)
335 png_bytep rp_stop
= row
+ row_info
->rowbytes
;
336 png_const_bytep pp
= prev_row
;
338 uint8x8_t vlast
= vdup_n_u8(0);
340 vdest
.val
[3] = vdup_n_u8(0);
342 for (; rp
< rp_stop
; rp
+= 16, pp
+= 16)
345 uint8x8x4_t
*vrpt
, *vppt
;
346 uint8x8x4_t vrp
, vpp
;
347 uint32x2x4_t
*temp_pointer
;
349 vtmp
= vld4_u32(png_ptr(uint32_t,rp
));
350 vrpt
= png_ptr(uint8x8x4_t
,&vtmp
);
352 vtmp
= vld4_u32(png_ptrc(uint32_t,pp
));
353 vppt
= png_ptr(uint8x8x4_t
,&vtmp
);
356 vdest
.val
[0] = paeth(vdest
.val
[3], vpp
.val
[0], vlast
);
357 vdest
.val
[0] = vadd_u8(vdest
.val
[0], vrp
.val
[0]);
358 vdest
.val
[1] = paeth(vdest
.val
[0], vpp
.val
[1], vpp
.val
[0]);
359 vdest
.val
[1] = vadd_u8(vdest
.val
[1], vrp
.val
[1]);
360 vdest
.val
[2] = paeth(vdest
.val
[1], vpp
.val
[2], vpp
.val
[1]);
361 vdest
.val
[2] = vadd_u8(vdest
.val
[2], vrp
.val
[2]);
362 vdest
.val
[3] = paeth(vdest
.val
[2], vpp
.val
[3], vpp
.val
[2]);
363 vdest
.val
[3] = vadd_u8(vdest
.val
[3], vrp
.val
[3]);
367 vst4_lane_u32(png_ptr(uint32_t,rp
), png_ldr(uint32x2x4_t
,&vdest
), 0);
371 #endif /* PNG_ARM_NEON_OPT > 0 */
372 #endif /* PNG_ARM_NEON_IMPLEMENTATION == 1 (intrinsics) */