1 /* Medium-level subroutines: convert bit-field store and extract
2 and shifts, multiplies and divides to rtl instructions.
3 Copyright (C) 1987-2022 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Work around tree-optimization/91825. */
22 #pragma GCC diagnostic warning "-Wmaybe-uninitialized"
26 #include "coretypes.h"
38 #include "diagnostic-core.h"
39 #include "fold-const.h"
40 #include "stor-layout.h"
44 #include "langhooks.h"
45 #include "tree-vector-builder.h"
47 struct target_expmed default_target_expmed
;
49 struct target_expmed
*this_target_expmed
= &default_target_expmed
;
52 static bool store_integral_bit_field (rtx
, opt_scalar_int_mode
,
53 unsigned HOST_WIDE_INT
,
54 unsigned HOST_WIDE_INT
,
55 poly_uint64
, poly_uint64
,
56 machine_mode
, rtx
, bool, bool);
57 static void store_fixed_bit_field (rtx
, opt_scalar_int_mode
,
58 unsigned HOST_WIDE_INT
,
59 unsigned HOST_WIDE_INT
,
60 poly_uint64
, poly_uint64
,
61 rtx
, scalar_int_mode
, bool);
62 static void store_fixed_bit_field_1 (rtx
, scalar_int_mode
,
63 unsigned HOST_WIDE_INT
,
64 unsigned HOST_WIDE_INT
,
65 rtx
, scalar_int_mode
, bool);
66 static void store_split_bit_field (rtx
, opt_scalar_int_mode
,
67 unsigned HOST_WIDE_INT
,
68 unsigned HOST_WIDE_INT
,
69 poly_uint64
, poly_uint64
,
70 rtx
, scalar_int_mode
, bool);
71 static rtx
extract_integral_bit_field (rtx
, opt_scalar_int_mode
,
72 unsigned HOST_WIDE_INT
,
73 unsigned HOST_WIDE_INT
, int, rtx
,
74 machine_mode
, machine_mode
, bool, bool);
75 static rtx
extract_fixed_bit_field (machine_mode
, rtx
, opt_scalar_int_mode
,
76 unsigned HOST_WIDE_INT
,
77 unsigned HOST_WIDE_INT
, rtx
, int, bool);
78 static rtx
extract_fixed_bit_field_1 (machine_mode
, rtx
, scalar_int_mode
,
79 unsigned HOST_WIDE_INT
,
80 unsigned HOST_WIDE_INT
, rtx
, int, bool);
81 static rtx
lshift_value (machine_mode
, unsigned HOST_WIDE_INT
, int);
82 static rtx
extract_split_bit_field (rtx
, opt_scalar_int_mode
,
83 unsigned HOST_WIDE_INT
,
84 unsigned HOST_WIDE_INT
, int, bool);
85 static void do_cmp_and_jump (rtx
, rtx
, enum rtx_code
, machine_mode
, rtx_code_label
*);
86 static rtx
expand_smod_pow2 (scalar_int_mode
, rtx
, HOST_WIDE_INT
);
87 static rtx
expand_sdiv_pow2 (scalar_int_mode
, rtx
, HOST_WIDE_INT
);
89 /* Return a constant integer mask value of mode MODE with BITSIZE ones
90 followed by BITPOS zeros, or the complement of that if COMPLEMENT.
91 The mask is truncated if necessary to the width of mode MODE. The
92 mask is zero-extended if BITSIZE+BITPOS is too small for MODE. */
95 mask_rtx (scalar_int_mode mode
, int bitpos
, int bitsize
, bool complement
)
97 return immed_wide_int_const
98 (wi::shifted_mask (bitpos
, bitsize
, complement
,
99 GET_MODE_PRECISION (mode
)), mode
);
102 /* Test whether a value is zero of a power of two. */
103 #define EXACT_POWER_OF_2_OR_ZERO_P(x) \
104 (((x) & ((x) - HOST_WIDE_INT_1U)) == 0)
106 struct init_expmed_rtl
127 rtx pow2
[MAX_BITS_PER_WORD
];
128 rtx cint
[MAX_BITS_PER_WORD
];
132 init_expmed_one_conv (struct init_expmed_rtl
*all
, scalar_int_mode to_mode
,
133 scalar_int_mode from_mode
, bool speed
)
135 int to_size
, from_size
;
138 to_size
= GET_MODE_PRECISION (to_mode
);
139 from_size
= GET_MODE_PRECISION (from_mode
);
141 /* Most partial integers have a precision less than the "full"
142 integer it requires for storage. In case one doesn't, for
143 comparison purposes here, reduce the bit size by one in that
145 if (GET_MODE_CLASS (to_mode
) == MODE_PARTIAL_INT
146 && pow2p_hwi (to_size
))
148 if (GET_MODE_CLASS (from_mode
) == MODE_PARTIAL_INT
149 && pow2p_hwi (from_size
))
152 /* Assume cost of zero-extend and sign-extend is the same. */
153 which
= (to_size
< from_size
? all
->trunc
: all
->zext
);
155 PUT_MODE (all
->reg
, from_mode
);
156 set_convert_cost (to_mode
, from_mode
, speed
,
157 set_src_cost (which
, to_mode
, speed
));
158 /* Restore all->reg's mode. */
159 PUT_MODE (all
->reg
, to_mode
);
163 init_expmed_one_mode (struct init_expmed_rtl
*all
,
164 machine_mode mode
, int speed
)
166 int m
, n
, mode_bitsize
;
167 machine_mode mode_from
;
169 mode_bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
171 PUT_MODE (all
->reg
, mode
);
172 PUT_MODE (all
->plus
, mode
);
173 PUT_MODE (all
->neg
, mode
);
174 PUT_MODE (all
->mult
, mode
);
175 PUT_MODE (all
->sdiv
, mode
);
176 PUT_MODE (all
->udiv
, mode
);
177 PUT_MODE (all
->sdiv_32
, mode
);
178 PUT_MODE (all
->smod_32
, mode
);
179 PUT_MODE (all
->wide_trunc
, mode
);
180 PUT_MODE (all
->shift
, mode
);
181 PUT_MODE (all
->shift_mult
, mode
);
182 PUT_MODE (all
->shift_add
, mode
);
183 PUT_MODE (all
->shift_sub0
, mode
);
184 PUT_MODE (all
->shift_sub1
, mode
);
185 PUT_MODE (all
->zext
, mode
);
186 PUT_MODE (all
->trunc
, mode
);
188 set_add_cost (speed
, mode
, set_src_cost (all
->plus
, mode
, speed
));
189 set_neg_cost (speed
, mode
, set_src_cost (all
->neg
, mode
, speed
));
190 set_mul_cost (speed
, mode
, set_src_cost (all
->mult
, mode
, speed
));
191 set_sdiv_cost (speed
, mode
, set_src_cost (all
->sdiv
, mode
, speed
));
192 set_udiv_cost (speed
, mode
, set_src_cost (all
->udiv
, mode
, speed
));
194 set_sdiv_pow2_cheap (speed
, mode
, (set_src_cost (all
->sdiv_32
, mode
, speed
)
195 <= 2 * add_cost (speed
, mode
)));
196 set_smod_pow2_cheap (speed
, mode
, (set_src_cost (all
->smod_32
, mode
, speed
)
197 <= 4 * add_cost (speed
, mode
)));
199 set_shift_cost (speed
, mode
, 0, 0);
201 int cost
= add_cost (speed
, mode
);
202 set_shiftadd_cost (speed
, mode
, 0, cost
);
203 set_shiftsub0_cost (speed
, mode
, 0, cost
);
204 set_shiftsub1_cost (speed
, mode
, 0, cost
);
207 n
= MIN (MAX_BITS_PER_WORD
, mode_bitsize
);
208 for (m
= 1; m
< n
; m
++)
210 XEXP (all
->shift
, 1) = all
->cint
[m
];
211 XEXP (all
->shift_mult
, 1) = all
->pow2
[m
];
213 set_shift_cost (speed
, mode
, m
, set_src_cost (all
->shift
, mode
, speed
));
214 set_shiftadd_cost (speed
, mode
, m
, set_src_cost (all
->shift_add
, mode
,
216 set_shiftsub0_cost (speed
, mode
, m
, set_src_cost (all
->shift_sub0
, mode
,
218 set_shiftsub1_cost (speed
, mode
, m
, set_src_cost (all
->shift_sub1
, mode
,
222 scalar_int_mode int_mode_to
;
223 if (is_a
<scalar_int_mode
> (mode
, &int_mode_to
))
225 for (mode_from
= MIN_MODE_INT
; mode_from
<= MAX_MODE_INT
;
226 mode_from
= (machine_mode
)(mode_from
+ 1))
227 init_expmed_one_conv (all
, int_mode_to
,
228 as_a
<scalar_int_mode
> (mode_from
), speed
);
230 scalar_int_mode wider_mode
;
231 if (GET_MODE_CLASS (int_mode_to
) == MODE_INT
232 && GET_MODE_WIDER_MODE (int_mode_to
).exists (&wider_mode
))
234 PUT_MODE (all
->reg
, mode
);
235 PUT_MODE (all
->zext
, wider_mode
);
236 PUT_MODE (all
->wide_mult
, wider_mode
);
237 PUT_MODE (all
->wide_lshr
, wider_mode
);
238 XEXP (all
->wide_lshr
, 1)
239 = gen_int_shift_amount (wider_mode
, mode_bitsize
);
241 set_mul_widen_cost (speed
, wider_mode
,
242 set_src_cost (all
->wide_mult
, wider_mode
, speed
));
243 set_mul_highpart_cost (speed
, int_mode_to
,
244 set_src_cost (all
->wide_trunc
,
245 int_mode_to
, speed
));
253 struct init_expmed_rtl all
;
254 machine_mode mode
= QImode
;
257 memset (&all
, 0, sizeof all
);
258 for (m
= 1; m
< MAX_BITS_PER_WORD
; m
++)
260 all
.pow2
[m
] = GEN_INT (HOST_WIDE_INT_1
<< m
);
261 all
.cint
[m
] = GEN_INT (m
);
264 /* Avoid using hard regs in ways which may be unsupported. */
265 all
.reg
= gen_raw_REG (mode
, LAST_VIRTUAL_REGISTER
+ 1);
266 all
.plus
= gen_rtx_PLUS (mode
, all
.reg
, all
.reg
);
267 all
.neg
= gen_rtx_NEG (mode
, all
.reg
);
268 all
.mult
= gen_rtx_MULT (mode
, all
.reg
, all
.reg
);
269 all
.sdiv
= gen_rtx_DIV (mode
, all
.reg
, all
.reg
);
270 all
.udiv
= gen_rtx_UDIV (mode
, all
.reg
, all
.reg
);
271 all
.sdiv_32
= gen_rtx_DIV (mode
, all
.reg
, all
.pow2
[5]);
272 all
.smod_32
= gen_rtx_MOD (mode
, all
.reg
, all
.pow2
[5]);
273 all
.zext
= gen_rtx_ZERO_EXTEND (mode
, all
.reg
);
274 all
.wide_mult
= gen_rtx_MULT (mode
, all
.zext
, all
.zext
);
275 all
.wide_lshr
= gen_rtx_LSHIFTRT (mode
, all
.wide_mult
, all
.reg
);
276 all
.wide_trunc
= gen_rtx_TRUNCATE (mode
, all
.wide_lshr
);
277 all
.shift
= gen_rtx_ASHIFT (mode
, all
.reg
, all
.reg
);
278 all
.shift_mult
= gen_rtx_MULT (mode
, all
.reg
, all
.reg
);
279 all
.shift_add
= gen_rtx_PLUS (mode
, all
.shift_mult
, all
.reg
);
280 all
.shift_sub0
= gen_rtx_MINUS (mode
, all
.shift_mult
, all
.reg
);
281 all
.shift_sub1
= gen_rtx_MINUS (mode
, all
.reg
, all
.shift_mult
);
282 all
.trunc
= gen_rtx_TRUNCATE (mode
, all
.reg
);
284 for (speed
= 0; speed
< 2; speed
++)
286 crtl
->maybe_hot_insn_p
= speed
;
287 set_zero_cost (speed
, set_src_cost (const0_rtx
, mode
, speed
));
289 for (mode
= MIN_MODE_INT
; mode
<= MAX_MODE_INT
;
290 mode
= (machine_mode
)(mode
+ 1))
291 init_expmed_one_mode (&all
, mode
, speed
);
293 if (MIN_MODE_PARTIAL_INT
!= VOIDmode
)
294 for (mode
= MIN_MODE_PARTIAL_INT
; mode
<= MAX_MODE_PARTIAL_INT
;
295 mode
= (machine_mode
)(mode
+ 1))
296 init_expmed_one_mode (&all
, mode
, speed
);
298 if (MIN_MODE_VECTOR_INT
!= VOIDmode
)
299 for (mode
= MIN_MODE_VECTOR_INT
; mode
<= MAX_MODE_VECTOR_INT
;
300 mode
= (machine_mode
)(mode
+ 1))
301 init_expmed_one_mode (&all
, mode
, speed
);
304 if (alg_hash_used_p ())
306 struct alg_hash_entry
*p
= alg_hash_entry_ptr (0);
307 memset (p
, 0, sizeof (*p
) * NUM_ALG_HASH_ENTRIES
);
310 set_alg_hash_used_p (true);
311 default_rtl_profile ();
313 ggc_free (all
.trunc
);
314 ggc_free (all
.shift_sub1
);
315 ggc_free (all
.shift_sub0
);
316 ggc_free (all
.shift_add
);
317 ggc_free (all
.shift_mult
);
318 ggc_free (all
.shift
);
319 ggc_free (all
.wide_trunc
);
320 ggc_free (all
.wide_lshr
);
321 ggc_free (all
.wide_mult
);
323 ggc_free (all
.smod_32
);
324 ggc_free (all
.sdiv_32
);
333 /* Return an rtx representing minus the value of X.
334 MODE is the intended mode of the result,
335 useful if X is a CONST_INT. */
338 negate_rtx (machine_mode mode
, rtx x
)
340 rtx result
= simplify_unary_operation (NEG
, mode
, x
, mode
);
343 result
= expand_unop (mode
, neg_optab
, x
, NULL_RTX
, 0);
348 /* Whether reverse storage order is supported on the target. */
349 static int reverse_storage_order_supported
= -1;
351 /* Check whether reverse storage order is supported on the target. */
354 check_reverse_storage_order_support (void)
356 if (BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
358 reverse_storage_order_supported
= 0;
359 sorry ("reverse scalar storage order");
362 reverse_storage_order_supported
= 1;
365 /* Whether reverse FP storage order is supported on the target. */
366 static int reverse_float_storage_order_supported
= -1;
368 /* Check whether reverse FP storage order is supported on the target. */
371 check_reverse_float_storage_order_support (void)
373 if (FLOAT_WORDS_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
375 reverse_float_storage_order_supported
= 0;
376 sorry ("reverse floating-point scalar storage order");
379 reverse_float_storage_order_supported
= 1;
382 /* Return an rtx representing value of X with reverse storage order.
383 MODE is the intended mode of the result,
384 useful if X is a CONST_INT. */
387 flip_storage_order (machine_mode mode
, rtx x
)
389 scalar_int_mode int_mode
;
395 if (COMPLEX_MODE_P (mode
))
397 rtx real
= read_complex_part (x
, false);
398 rtx imag
= read_complex_part (x
, true);
400 real
= flip_storage_order (GET_MODE_INNER (mode
), real
);
401 imag
= flip_storage_order (GET_MODE_INNER (mode
), imag
);
403 return gen_rtx_CONCAT (mode
, real
, imag
);
406 if (UNLIKELY (reverse_storage_order_supported
< 0))
407 check_reverse_storage_order_support ();
409 if (!is_a
<scalar_int_mode
> (mode
, &int_mode
))
411 if (FLOAT_MODE_P (mode
)
412 && UNLIKELY (reverse_float_storage_order_supported
< 0))
413 check_reverse_float_storage_order_support ();
415 if (!int_mode_for_size (GET_MODE_PRECISION (mode
), 0).exists (&int_mode
)
416 || !targetm
.scalar_mode_supported_p (int_mode
))
418 sorry ("reverse storage order for %smode", GET_MODE_NAME (mode
));
421 x
= gen_lowpart (int_mode
, x
);
424 result
= simplify_unary_operation (BSWAP
, int_mode
, x
, int_mode
);
426 result
= expand_unop (int_mode
, bswap_optab
, x
, NULL_RTX
, 1);
428 if (int_mode
!= mode
)
429 result
= gen_lowpart (mode
, result
);
434 /* If MODE is set, adjust bitfield memory MEM so that it points to the
435 first unit of mode MODE that contains a bitfield of size BITSIZE at
436 bit position BITNUM. If MODE is not set, return a BLKmode reference
437 to every byte in the bitfield. Set *NEW_BITNUM to the bit position
438 of the field within the new memory. */
441 narrow_bit_field_mem (rtx mem
, opt_scalar_int_mode mode
,
442 unsigned HOST_WIDE_INT bitsize
,
443 unsigned HOST_WIDE_INT bitnum
,
444 unsigned HOST_WIDE_INT
*new_bitnum
)
446 scalar_int_mode imode
;
447 if (mode
.exists (&imode
))
449 unsigned int unit
= GET_MODE_BITSIZE (imode
);
450 *new_bitnum
= bitnum
% unit
;
451 HOST_WIDE_INT offset
= (bitnum
- *new_bitnum
) / BITS_PER_UNIT
;
452 return adjust_bitfield_address (mem
, imode
, offset
);
456 *new_bitnum
= bitnum
% BITS_PER_UNIT
;
457 HOST_WIDE_INT offset
= bitnum
/ BITS_PER_UNIT
;
458 HOST_WIDE_INT size
= ((*new_bitnum
+ bitsize
+ BITS_PER_UNIT
- 1)
460 return adjust_bitfield_address_size (mem
, BLKmode
, offset
, size
);
464 /* The caller wants to perform insertion or extraction PATTERN on a
465 bitfield of size BITSIZE at BITNUM bits into memory operand OP0.
466 BITREGION_START and BITREGION_END are as for store_bit_field
467 and FIELDMODE is the natural mode of the field.
469 Search for a mode that is compatible with the memory access
470 restrictions and (where applicable) with a register insertion or
471 extraction. Return the new memory on success, storing the adjusted
472 bit position in *NEW_BITNUM. Return null otherwise. */
475 adjust_bit_field_mem_for_reg (enum extraction_pattern pattern
,
476 rtx op0
, HOST_WIDE_INT bitsize
,
477 HOST_WIDE_INT bitnum
,
478 poly_uint64 bitregion_start
,
479 poly_uint64 bitregion_end
,
480 machine_mode fieldmode
,
481 unsigned HOST_WIDE_INT
*new_bitnum
)
483 bit_field_mode_iterator
iter (bitsize
, bitnum
, bitregion_start
,
484 bitregion_end
, MEM_ALIGN (op0
),
485 MEM_VOLATILE_P (op0
));
486 scalar_int_mode best_mode
;
487 if (iter
.next_mode (&best_mode
))
489 /* We can use a memory in BEST_MODE. See whether this is true for
490 any wider modes. All other things being equal, we prefer to
491 use the widest mode possible because it tends to expose more
492 CSE opportunities. */
493 if (!iter
.prefer_smaller_modes ())
495 /* Limit the search to the mode required by the corresponding
496 register insertion or extraction instruction, if any. */
497 scalar_int_mode limit_mode
= word_mode
;
498 extraction_insn insn
;
499 if (get_best_reg_extraction_insn (&insn
, pattern
,
500 GET_MODE_BITSIZE (best_mode
),
502 limit_mode
= insn
.field_mode
;
504 scalar_int_mode wider_mode
;
505 while (iter
.next_mode (&wider_mode
)
506 && GET_MODE_SIZE (wider_mode
) <= GET_MODE_SIZE (limit_mode
))
507 best_mode
= wider_mode
;
509 return narrow_bit_field_mem (op0
, best_mode
, bitsize
, bitnum
,
515 /* Return true if a bitfield of size BITSIZE at bit number BITNUM within
516 a structure of mode STRUCT_MODE represents a lowpart subreg. The subreg
517 offset is then BITNUM / BITS_PER_UNIT. */
520 lowpart_bit_field_p (poly_uint64 bitnum
, poly_uint64 bitsize
,
521 machine_mode struct_mode
)
523 poly_uint64 regsize
= REGMODE_NATURAL_SIZE (struct_mode
);
524 if (BYTES_BIG_ENDIAN
)
525 return (multiple_p (bitnum
, BITS_PER_UNIT
)
526 && (known_eq (bitnum
+ bitsize
, GET_MODE_BITSIZE (struct_mode
))
527 || multiple_p (bitnum
+ bitsize
,
528 regsize
* BITS_PER_UNIT
)));
530 return multiple_p (bitnum
, regsize
* BITS_PER_UNIT
);
533 /* Return true if -fstrict-volatile-bitfields applies to an access of OP0
534 containing BITSIZE bits starting at BITNUM, with field mode FIELDMODE.
535 Return false if the access would touch memory outside the range
536 BITREGION_START to BITREGION_END for conformance to the C++ memory
540 strict_volatile_bitfield_p (rtx op0
, unsigned HOST_WIDE_INT bitsize
,
541 unsigned HOST_WIDE_INT bitnum
,
542 scalar_int_mode fieldmode
,
543 poly_uint64 bitregion_start
,
544 poly_uint64 bitregion_end
)
546 unsigned HOST_WIDE_INT modesize
= GET_MODE_BITSIZE (fieldmode
);
548 /* -fstrict-volatile-bitfields must be enabled and we must have a
551 || !MEM_VOLATILE_P (op0
)
552 || flag_strict_volatile_bitfields
<= 0)
555 /* The bit size must not be larger than the field mode, and
556 the field mode must not be larger than a word. */
557 if (bitsize
> modesize
|| modesize
> BITS_PER_WORD
)
560 /* Check for cases of unaligned fields that must be split. */
561 if (bitnum
% modesize
+ bitsize
> modesize
)
564 /* The memory must be sufficiently aligned for a MODESIZE access.
565 This condition guarantees, that the memory access will not
566 touch anything after the end of the structure. */
567 if (MEM_ALIGN (op0
) < modesize
)
570 /* Check for cases where the C++ memory model applies. */
571 if (maybe_ne (bitregion_end
, 0U)
572 && (maybe_lt (bitnum
- bitnum
% modesize
, bitregion_start
)
573 || maybe_gt (bitnum
- bitnum
% modesize
+ modesize
- 1,
580 /* Return true if OP is a memory and if a bitfield of size BITSIZE at
581 bit number BITNUM can be treated as a simple value of mode MODE.
582 Store the byte offset in *BYTENUM if so. */
585 simple_mem_bitfield_p (rtx op0
, poly_uint64 bitsize
, poly_uint64 bitnum
,
586 machine_mode mode
, poly_uint64
*bytenum
)
589 && multiple_p (bitnum
, BITS_PER_UNIT
, bytenum
)
590 && known_eq (bitsize
, GET_MODE_BITSIZE (mode
))
591 && (!targetm
.slow_unaligned_access (mode
, MEM_ALIGN (op0
))
592 || (multiple_p (bitnum
, GET_MODE_ALIGNMENT (mode
))
593 && MEM_ALIGN (op0
) >= GET_MODE_ALIGNMENT (mode
))));
596 /* Try to use instruction INSV to store VALUE into a field of OP0.
597 If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is a
598 BLKmode MEM. VALUE_MODE is the mode of VALUE. BITSIZE and BITNUM
599 are as for store_bit_field. */
602 store_bit_field_using_insv (const extraction_insn
*insv
, rtx op0
,
603 opt_scalar_int_mode op0_mode
,
604 unsigned HOST_WIDE_INT bitsize
,
605 unsigned HOST_WIDE_INT bitnum
,
606 rtx value
, scalar_int_mode value_mode
)
608 class expand_operand ops
[4];
611 rtx_insn
*last
= get_last_insn ();
612 bool copy_back
= false;
614 scalar_int_mode op_mode
= insv
->field_mode
;
615 unsigned int unit
= GET_MODE_BITSIZE (op_mode
);
616 if (bitsize
== 0 || bitsize
> unit
)
620 /* Get a reference to the first byte of the field. */
621 xop0
= narrow_bit_field_mem (xop0
, insv
->struct_mode
, bitsize
, bitnum
,
625 /* Convert from counting within OP0 to counting in OP_MODE. */
626 if (BYTES_BIG_ENDIAN
)
627 bitnum
+= unit
- GET_MODE_BITSIZE (op0_mode
.require ());
629 /* If xop0 is a register, we need it in OP_MODE
630 to make it acceptable to the format of insv. */
631 if (GET_CODE (xop0
) == SUBREG
)
633 /* If such a SUBREG can't be created, give up. */
634 if (!validate_subreg (op_mode
, GET_MODE (SUBREG_REG (xop0
)),
635 SUBREG_REG (xop0
), SUBREG_BYTE (xop0
)))
637 /* We can't just change the mode, because this might clobber op0,
638 and we will need the original value of op0 if insv fails. */
639 xop0
= gen_rtx_SUBREG (op_mode
, SUBREG_REG (xop0
),
642 if (REG_P (xop0
) && GET_MODE (xop0
) != op_mode
)
643 xop0
= gen_lowpart_SUBREG (op_mode
, xop0
);
646 /* If the destination is a paradoxical subreg such that we need a
647 truncate to the inner mode, perform the insertion on a temporary and
648 truncate the result to the original destination. Note that we can't
649 just truncate the paradoxical subreg as (truncate:N (subreg:W (reg:N
650 X) 0)) is (reg:N X). */
651 if (GET_CODE (xop0
) == SUBREG
652 && REG_P (SUBREG_REG (xop0
))
653 && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (SUBREG_REG (xop0
)),
656 rtx tem
= gen_reg_rtx (op_mode
);
657 emit_move_insn (tem
, xop0
);
662 /* There are similar overflow check at the start of store_bit_field_1,
663 but that only check the situation where the field lies completely
664 outside the register, while there do have situation where the field
665 lies partialy in the register, we need to adjust bitsize for this
666 partial overflow situation. Without this fix, pr48335-2.c on big-endian
667 will broken on those arch support bit insert instruction, like arm, aarch64
669 if (bitsize
+ bitnum
> unit
&& bitnum
< unit
)
671 warning (OPT_Wextra
, "write of %wu-bit data outside the bound of "
672 "destination object, data truncated into %wu-bit",
673 bitsize
, unit
- bitnum
);
674 bitsize
= unit
- bitnum
;
677 /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
678 "backwards" from the size of the unit we are inserting into.
679 Otherwise, we count bits from the most significant on a
680 BYTES/BITS_BIG_ENDIAN machine. */
682 if (BITS_BIG_ENDIAN
!= BYTES_BIG_ENDIAN
)
683 bitnum
= unit
- bitsize
- bitnum
;
685 /* Convert VALUE to op_mode (which insv insn wants) in VALUE1. */
687 if (value_mode
!= op_mode
)
689 if (GET_MODE_BITSIZE (value_mode
) >= bitsize
)
692 /* Optimization: Don't bother really extending VALUE
693 if it has all the bits we will actually use. However,
694 if we must narrow it, be sure we do it correctly. */
696 if (GET_MODE_SIZE (value_mode
) < GET_MODE_SIZE (op_mode
))
698 tmp
= simplify_subreg (op_mode
, value1
, value_mode
, 0);
700 tmp
= simplify_gen_subreg (op_mode
,
701 force_reg (value_mode
, value1
),
706 tmp
= gen_lowpart_if_possible (op_mode
, value1
);
708 tmp
= gen_lowpart (op_mode
, force_reg (value_mode
, value1
));
712 else if (CONST_INT_P (value
))
713 value1
= gen_int_mode (INTVAL (value
), op_mode
);
715 /* Parse phase is supposed to make VALUE's data type
716 match that of the component reference, which is a type
717 at least as wide as the field; so VALUE should have
718 a mode that corresponds to that type. */
719 gcc_assert (CONSTANT_P (value
));
722 create_fixed_operand (&ops
[0], xop0
);
723 create_integer_operand (&ops
[1], bitsize
);
724 create_integer_operand (&ops
[2], bitnum
);
725 create_input_operand (&ops
[3], value1
, op_mode
);
726 if (maybe_expand_insn (insv
->icode
, 4, ops
))
729 convert_move (op0
, xop0
, true);
732 delete_insns_since (last
);
736 /* A subroutine of store_bit_field, with the same arguments. Return true
737 if the operation could be implemented.
739 If FALLBACK_P is true, fall back to store_fixed_bit_field if we have
740 no other way of implementing the operation. If FALLBACK_P is false,
741 return false instead.
743 if UNDEFINED_P is true then STR_RTX is undefined and may be set using
747 store_bit_field_1 (rtx str_rtx
, poly_uint64 bitsize
, poly_uint64 bitnum
,
748 poly_uint64 bitregion_start
, poly_uint64 bitregion_end
,
749 machine_mode fieldmode
,
750 rtx value
, bool reverse
, bool fallback_p
, bool undefined_p
)
754 while (GET_CODE (op0
) == SUBREG
)
756 bitnum
+= subreg_memory_offset (op0
) * BITS_PER_UNIT
;
757 op0
= SUBREG_REG (op0
);
760 /* No action is needed if the target is a register and if the field
761 lies completely outside that register. This can occur if the source
762 code contains an out-of-bounds access to a small array. */
763 if (REG_P (op0
) && known_ge (bitnum
, GET_MODE_BITSIZE (GET_MODE (op0
))))
766 /* Use vec_set patterns for inserting parts of vectors whenever
768 machine_mode outermode
= GET_MODE (op0
);
769 scalar_mode innermode
= GET_MODE_INNER (outermode
);
771 if (VECTOR_MODE_P (outermode
)
773 && optab_handler (vec_set_optab
, outermode
) != CODE_FOR_nothing
774 && fieldmode
== innermode
775 && known_eq (bitsize
, GET_MODE_BITSIZE (innermode
))
776 && multiple_p (bitnum
, GET_MODE_BITSIZE (innermode
), &pos
))
778 class expand_operand ops
[3];
779 enum insn_code icode
= optab_handler (vec_set_optab
, outermode
);
781 create_fixed_operand (&ops
[0], op0
);
782 create_input_operand (&ops
[1], value
, innermode
);
783 create_integer_operand (&ops
[2], pos
);
784 if (maybe_expand_insn (icode
, 3, ops
))
788 /* If the target is a register, overwriting the entire object, or storing
789 a full-word or multi-word field can be done with just a SUBREG. */
791 && known_eq (bitsize
, GET_MODE_BITSIZE (fieldmode
)))
793 /* Use the subreg machinery either to narrow OP0 to the required
794 words or to cope with mode punning between equal-sized modes.
795 In the latter case, use subreg on the rhs side, not lhs. */
798 poly_uint64 regsize
= REGMODE_NATURAL_SIZE (GET_MODE (op0
));
799 if (known_eq (bitnum
, 0U)
800 && known_eq (bitsize
, GET_MODE_BITSIZE (GET_MODE (op0
))))
802 sub
= simplify_gen_subreg (GET_MODE (op0
), value
, fieldmode
, 0);
806 sub
= flip_storage_order (GET_MODE (op0
), sub
);
807 emit_move_insn (op0
, sub
);
811 else if (multiple_p (bitnum
, BITS_PER_UNIT
, &bytenum
)
813 || (multiple_p (bitnum
, regsize
* BITS_PER_UNIT
)
814 && multiple_p (bitsize
, regsize
* BITS_PER_UNIT
)))
815 && known_ge (GET_MODE_BITSIZE (GET_MODE (op0
)), bitsize
))
817 sub
= simplify_gen_subreg (fieldmode
, op0
, GET_MODE (op0
), bytenum
);
821 value
= flip_storage_order (fieldmode
, value
);
822 emit_move_insn (sub
, value
);
828 /* If the target is memory, storing any naturally aligned field can be
829 done with a simple store. For targets that support fast unaligned
830 memory, any naturally sized, unit aligned field can be done directly. */
832 if (simple_mem_bitfield_p (op0
, bitsize
, bitnum
, fieldmode
, &bytenum
))
834 op0
= adjust_bitfield_address (op0
, fieldmode
, bytenum
);
836 value
= flip_storage_order (fieldmode
, value
);
837 emit_move_insn (op0
, value
);
841 /* It's possible we'll need to handle other cases here for
842 polynomial bitnum and bitsize. */
844 /* From here on we need to be looking at a fixed-size insertion. */
845 unsigned HOST_WIDE_INT ibitsize
= bitsize
.to_constant ();
846 unsigned HOST_WIDE_INT ibitnum
= bitnum
.to_constant ();
848 /* Make sure we are playing with integral modes. Pun with subregs
849 if we aren't. This must come after the entire register case above,
850 since that case is valid for any mode. The following cases are only
851 valid for integral modes. */
852 opt_scalar_int_mode op0_mode
= int_mode_for_mode (GET_MODE (op0
));
853 scalar_int_mode imode
;
854 if (!op0_mode
.exists (&imode
) || imode
!= GET_MODE (op0
))
857 op0
= adjust_bitfield_address_size (op0
, op0_mode
.else_blk (),
859 else if (!op0_mode
.exists ())
862 && known_eq (ibitsize
, GET_MODE_BITSIZE (GET_MODE (op0
)))
866 value
= adjust_address (value
, GET_MODE (op0
), 0);
867 emit_move_insn (op0
, value
);
872 rtx temp
= assign_stack_temp (GET_MODE (op0
),
873 GET_MODE_SIZE (GET_MODE (op0
)));
874 emit_move_insn (temp
, op0
);
875 store_bit_field_1 (temp
, bitsize
, bitnum
, 0, 0, fieldmode
, value
,
876 reverse
, fallback_p
, undefined_p
);
877 emit_move_insn (op0
, temp
);
881 op0
= gen_lowpart (op0_mode
.require (), op0
);
884 return store_integral_bit_field (op0
, op0_mode
, ibitsize
, ibitnum
,
885 bitregion_start
, bitregion_end
,
886 fieldmode
, value
, reverse
, fallback_p
);
889 /* Subroutine of store_bit_field_1, with the same arguments, except
890 that BITSIZE and BITNUM are constant. Handle cases specific to
891 integral modes. If OP0_MODE is defined, it is the mode of OP0,
892 otherwise OP0 is a BLKmode MEM. */
895 store_integral_bit_field (rtx op0
, opt_scalar_int_mode op0_mode
,
896 unsigned HOST_WIDE_INT bitsize
,
897 unsigned HOST_WIDE_INT bitnum
,
898 poly_uint64 bitregion_start
,
899 poly_uint64 bitregion_end
,
900 machine_mode fieldmode
,
901 rtx value
, bool reverse
, bool fallback_p
)
903 /* Storing an lsb-aligned field in a register
904 can be done with a movstrict instruction. */
908 && lowpart_bit_field_p (bitnum
, bitsize
, op0_mode
.require ())
909 && known_eq (bitsize
, GET_MODE_BITSIZE (fieldmode
))
910 && optab_handler (movstrict_optab
, fieldmode
) != CODE_FOR_nothing
)
912 class expand_operand ops
[2];
913 enum insn_code icode
= optab_handler (movstrict_optab
, fieldmode
);
915 unsigned HOST_WIDE_INT subreg_off
;
917 if (GET_CODE (arg0
) == SUBREG
)
919 /* Else we've got some float mode source being extracted into
920 a different float mode destination -- this combination of
921 subregs results in Severe Tire Damage. */
922 gcc_assert (GET_MODE (SUBREG_REG (arg0
)) == fieldmode
923 || GET_MODE_CLASS (fieldmode
) == MODE_INT
924 || GET_MODE_CLASS (fieldmode
) == MODE_PARTIAL_INT
);
925 arg0
= SUBREG_REG (arg0
);
928 subreg_off
= bitnum
/ BITS_PER_UNIT
;
929 if (validate_subreg (fieldmode
, GET_MODE (arg0
), arg0
, subreg_off
)
930 /* STRICT_LOW_PART must have a non-paradoxical subreg as
932 && !paradoxical_subreg_p (fieldmode
, GET_MODE (arg0
)))
934 arg0
= gen_rtx_SUBREG (fieldmode
, arg0
, subreg_off
);
936 create_fixed_operand (&ops
[0], arg0
);
937 /* Shrink the source operand to FIELDMODE. */
938 create_convert_operand_to (&ops
[1], value
, fieldmode
, false);
939 if (maybe_expand_insn (icode
, 2, ops
))
944 /* Handle fields bigger than a word. */
946 if (bitsize
> BITS_PER_WORD
)
948 /* Here we transfer the words of the field
949 in the order least significant first.
950 This is because the most significant word is the one which may
952 However, only do that if the value is not BLKmode. */
954 const bool backwards
= WORDS_BIG_ENDIAN
&& fieldmode
!= BLKmode
;
955 const int nwords
= (bitsize
+ (BITS_PER_WORD
- 1)) / BITS_PER_WORD
;
958 /* This is the mode we must force value to, so that there will be enough
959 subwords to extract. Note that fieldmode will often (always?) be
960 VOIDmode, because that is what store_field uses to indicate that this
961 is a bit field, but passing VOIDmode to operand_subword_force
964 The mode must be fixed-size, since insertions into variable-sized
965 objects are meant to be handled before calling this function. */
966 fixed_size_mode value_mode
= as_a
<fixed_size_mode
> (GET_MODE (value
));
967 if (value_mode
== VOIDmode
)
968 value_mode
= smallest_int_mode_for_size (nwords
* BITS_PER_WORD
);
970 last
= get_last_insn ();
971 for (int i
= 0; i
< nwords
; i
++)
973 /* Number of bits to be stored in this iteration, i.e. BITS_PER_WORD
974 except maybe for the last iteration. */
975 const unsigned HOST_WIDE_INT new_bitsize
976 = MIN (BITS_PER_WORD
, bitsize
- i
* BITS_PER_WORD
);
977 /* Bit offset from the starting bit number in the target. */
978 const unsigned int bit_offset
979 = backwards
^ reverse
980 ? MAX ((int) bitsize
- (i
+ 1) * BITS_PER_WORD
, 0)
982 /* Starting word number in the value. */
983 const unsigned int wordnum
985 ? GET_MODE_SIZE (value_mode
) / UNITS_PER_WORD
- (i
+ 1)
987 /* The chunk of the value in word_mode. We use bit-field extraction
988 in BLKmode to handle unaligned memory references and to shift the
989 last chunk right on big-endian machines if need be. */
991 = fieldmode
== BLKmode
992 ? extract_bit_field (value
, new_bitsize
, wordnum
* BITS_PER_WORD
,
993 1, NULL_RTX
, word_mode
, word_mode
, false,
995 : operand_subword_force (value
, wordnum
, value_mode
);
997 if (!store_bit_field_1 (op0
, new_bitsize
,
999 bitregion_start
, bitregion_end
,
1001 value_word
, reverse
, fallback_p
, false))
1003 delete_insns_since (last
);
1010 /* If VALUE has a floating-point or complex mode, access it as an
1011 integer of the corresponding size. This can occur on a machine
1012 with 64 bit registers that uses SFmode for float. It can also
1013 occur for unaligned float or complex fields. */
1014 rtx orig_value
= value
;
1015 scalar_int_mode value_mode
;
1016 if (GET_MODE (value
) == VOIDmode
)
1017 /* By this point we've dealt with values that are bigger than a word,
1018 so word_mode is a conservatively correct choice. */
1019 value_mode
= word_mode
;
1020 else if (!is_a
<scalar_int_mode
> (GET_MODE (value
), &value_mode
))
1022 value_mode
= int_mode_for_mode (GET_MODE (value
)).require ();
1023 value
= gen_reg_rtx (value_mode
);
1024 emit_move_insn (gen_lowpart (GET_MODE (orig_value
), value
), orig_value
);
1027 /* If OP0 is a multi-word register, narrow it to the affected word.
1028 If the region spans two words, defer to store_split_bit_field.
1029 Don't do this if op0 is a single hard register wider than word
1030 such as a float or vector register. */
1032 && GET_MODE_SIZE (op0_mode
.require ()) > UNITS_PER_WORD
1034 || !HARD_REGISTER_P (op0
)
1035 || hard_regno_nregs (REGNO (op0
), op0_mode
.require ()) != 1))
1037 if (bitnum
% BITS_PER_WORD
+ bitsize
> BITS_PER_WORD
)
1042 store_split_bit_field (op0
, op0_mode
, bitsize
, bitnum
,
1043 bitregion_start
, bitregion_end
,
1044 value
, value_mode
, reverse
);
1047 op0
= simplify_gen_subreg (word_mode
, op0
, op0_mode
.require (),
1048 bitnum
/ BITS_PER_WORD
* UNITS_PER_WORD
);
1050 op0_mode
= word_mode
;
1051 bitnum
%= BITS_PER_WORD
;
1054 /* From here on we can assume that the field to be stored in fits
1055 within a word. If the destination is a register, it too fits
1058 extraction_insn insv
;
1061 && get_best_reg_extraction_insn (&insv
, EP_insv
,
1062 GET_MODE_BITSIZE (op0_mode
.require ()),
1064 && store_bit_field_using_insv (&insv
, op0
, op0_mode
,
1065 bitsize
, bitnum
, value
, value_mode
))
1068 /* If OP0 is a memory, try copying it to a register and seeing if a
1069 cheap register alternative is available. */
1070 if (MEM_P (op0
) && !reverse
)
1072 if (get_best_mem_extraction_insn (&insv
, EP_insv
, bitsize
, bitnum
,
1074 && store_bit_field_using_insv (&insv
, op0
, op0_mode
,
1075 bitsize
, bitnum
, value
, value_mode
))
1078 rtx_insn
*last
= get_last_insn ();
1080 /* Try loading part of OP0 into a register, inserting the bitfield
1081 into that, and then copying the result back to OP0. */
1082 unsigned HOST_WIDE_INT bitpos
;
1083 rtx xop0
= adjust_bit_field_mem_for_reg (EP_insv
, op0
, bitsize
, bitnum
,
1084 bitregion_start
, bitregion_end
,
1085 fieldmode
, &bitpos
);
1088 rtx tempreg
= copy_to_reg (xop0
);
1089 if (store_bit_field_1 (tempreg
, bitsize
, bitpos
,
1090 bitregion_start
, bitregion_end
,
1091 fieldmode
, orig_value
, reverse
, false, false))
1093 emit_move_insn (xop0
, tempreg
);
1096 delete_insns_since (last
);
1103 store_fixed_bit_field (op0
, op0_mode
, bitsize
, bitnum
, bitregion_start
,
1104 bitregion_end
, value
, value_mode
, reverse
);
1108 /* Generate code to store value from rtx VALUE
1109 into a bit-field within structure STR_RTX
1110 containing BITSIZE bits starting at bit BITNUM.
1112 BITREGION_START is bitpos of the first bitfield in this region.
1113 BITREGION_END is the bitpos of the ending bitfield in this region.
1114 These two fields are 0, if the C++ memory model does not apply,
1115 or we are not interested in keeping track of bitfield regions.
1117 FIELDMODE is the machine-mode of the FIELD_DECL node for this field.
1119 If REVERSE is true, the store is to be done in reverse order.
1121 If UNDEFINED_P is true then STR_RTX is currently undefined. */
1124 store_bit_field (rtx str_rtx
, poly_uint64 bitsize
, poly_uint64 bitnum
,
1125 poly_uint64 bitregion_start
, poly_uint64 bitregion_end
,
1126 machine_mode fieldmode
,
1127 rtx value
, bool reverse
, bool undefined_p
)
1129 /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
1130 unsigned HOST_WIDE_INT ibitsize
= 0, ibitnum
= 0;
1131 scalar_int_mode int_mode
;
1132 if (bitsize
.is_constant (&ibitsize
)
1133 && bitnum
.is_constant (&ibitnum
)
1134 && is_a
<scalar_int_mode
> (fieldmode
, &int_mode
)
1135 && strict_volatile_bitfield_p (str_rtx
, ibitsize
, ibitnum
, int_mode
,
1136 bitregion_start
, bitregion_end
))
1138 /* Storing of a full word can be done with a simple store.
1139 We know here that the field can be accessed with one single
1140 instruction. For targets that support unaligned memory,
1141 an unaligned access may be necessary. */
1142 if (ibitsize
== GET_MODE_BITSIZE (int_mode
))
1144 str_rtx
= adjust_bitfield_address (str_rtx
, int_mode
,
1145 ibitnum
/ BITS_PER_UNIT
);
1147 value
= flip_storage_order (int_mode
, value
);
1148 gcc_assert (ibitnum
% BITS_PER_UNIT
== 0);
1149 emit_move_insn (str_rtx
, value
);
1155 str_rtx
= narrow_bit_field_mem (str_rtx
, int_mode
, ibitsize
,
1157 gcc_assert (ibitnum
+ ibitsize
<= GET_MODE_BITSIZE (int_mode
));
1158 temp
= copy_to_reg (str_rtx
);
1159 if (!store_bit_field_1 (temp
, ibitsize
, ibitnum
, 0, 0,
1160 int_mode
, value
, reverse
, true, undefined_p
))
1163 emit_move_insn (str_rtx
, temp
);
1169 /* Under the C++0x memory model, we must not touch bits outside the
1170 bit region. Adjust the address to start at the beginning of the
1172 if (MEM_P (str_rtx
) && maybe_ne (bitregion_start
, 0U))
1174 scalar_int_mode best_mode
;
1175 machine_mode addr_mode
= VOIDmode
;
1177 poly_uint64 offset
= exact_div (bitregion_start
, BITS_PER_UNIT
);
1178 bitnum
-= bitregion_start
;
1179 poly_int64 size
= bits_to_bytes_round_up (bitnum
+ bitsize
);
1180 bitregion_end
-= bitregion_start
;
1181 bitregion_start
= 0;
1182 if (bitsize
.is_constant (&ibitsize
)
1183 && bitnum
.is_constant (&ibitnum
)
1184 && get_best_mode (ibitsize
, ibitnum
,
1185 bitregion_start
, bitregion_end
,
1186 MEM_ALIGN (str_rtx
), INT_MAX
,
1187 MEM_VOLATILE_P (str_rtx
), &best_mode
))
1188 addr_mode
= best_mode
;
1189 str_rtx
= adjust_bitfield_address_size (str_rtx
, addr_mode
,
1193 if (!store_bit_field_1 (str_rtx
, bitsize
, bitnum
,
1194 bitregion_start
, bitregion_end
,
1195 fieldmode
, value
, reverse
, true, undefined_p
))
1199 /* Use shifts and boolean operations to store VALUE into a bit field of
1200 width BITSIZE in OP0, starting at bit BITNUM. If OP0_MODE is defined,
1201 it is the mode of OP0, otherwise OP0 is a BLKmode MEM. VALUE_MODE is
1204 If REVERSE is true, the store is to be done in reverse order. */
1207 store_fixed_bit_field (rtx op0
, opt_scalar_int_mode op0_mode
,
1208 unsigned HOST_WIDE_INT bitsize
,
1209 unsigned HOST_WIDE_INT bitnum
,
1210 poly_uint64 bitregion_start
, poly_uint64 bitregion_end
,
1211 rtx value
, scalar_int_mode value_mode
, bool reverse
)
1213 /* There is a case not handled here:
1214 a structure with a known alignment of just a halfword
1215 and a field split across two aligned halfwords within the structure.
1216 Or likewise a structure with a known alignment of just a byte
1217 and a field split across two bytes.
1218 Such cases are not supposed to be able to occur. */
1220 scalar_int_mode best_mode
;
1223 unsigned int max_bitsize
= BITS_PER_WORD
;
1224 scalar_int_mode imode
;
1225 if (op0_mode
.exists (&imode
) && GET_MODE_BITSIZE (imode
) < max_bitsize
)
1226 max_bitsize
= GET_MODE_BITSIZE (imode
);
1228 if (!get_best_mode (bitsize
, bitnum
, bitregion_start
, bitregion_end
,
1229 MEM_ALIGN (op0
), max_bitsize
, MEM_VOLATILE_P (op0
),
1232 /* The only way this should occur is if the field spans word
1234 store_split_bit_field (op0
, op0_mode
, bitsize
, bitnum
,
1235 bitregion_start
, bitregion_end
,
1236 value
, value_mode
, reverse
);
1240 op0
= narrow_bit_field_mem (op0
, best_mode
, bitsize
, bitnum
, &bitnum
);
1243 best_mode
= op0_mode
.require ();
1245 store_fixed_bit_field_1 (op0
, best_mode
, bitsize
, bitnum
,
1246 value
, value_mode
, reverse
);
1249 /* Helper function for store_fixed_bit_field, stores
1250 the bit field always using MODE, which is the mode of OP0. The other
1251 arguments are as for store_fixed_bit_field. */
1254 store_fixed_bit_field_1 (rtx op0
, scalar_int_mode mode
,
1255 unsigned HOST_WIDE_INT bitsize
,
1256 unsigned HOST_WIDE_INT bitnum
,
1257 rtx value
, scalar_int_mode value_mode
, bool reverse
)
1263 /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
1264 for invalid input, such as f5 from gcc.dg/pr48335-2.c. */
1266 if (reverse
? !BYTES_BIG_ENDIAN
: BYTES_BIG_ENDIAN
)
1267 /* BITNUM is the distance between our msb
1268 and that of the containing datum.
1269 Convert it to the distance from the lsb. */
1270 bitnum
= GET_MODE_BITSIZE (mode
) - bitsize
- bitnum
;
1272 /* Now BITNUM is always the distance between our lsb
1275 /* Shift VALUE left by BITNUM bits. If VALUE is not constant,
1276 we must first convert its mode to MODE. */
1278 if (CONST_INT_P (value
))
1280 unsigned HOST_WIDE_INT v
= UINTVAL (value
);
1282 if (bitsize
< HOST_BITS_PER_WIDE_INT
)
1283 v
&= (HOST_WIDE_INT_1U
<< bitsize
) - 1;
1287 else if ((bitsize
< HOST_BITS_PER_WIDE_INT
1288 && v
== (HOST_WIDE_INT_1U
<< bitsize
) - 1)
1289 || (bitsize
== HOST_BITS_PER_WIDE_INT
1290 && v
== HOST_WIDE_INT_M1U
))
1293 value
= lshift_value (mode
, v
, bitnum
);
1297 int must_and
= (GET_MODE_BITSIZE (value_mode
) != bitsize
1298 && bitnum
+ bitsize
!= GET_MODE_BITSIZE (mode
));
1300 if (value_mode
!= mode
)
1301 value
= convert_to_mode (mode
, value
, 1);
1304 value
= expand_binop (mode
, and_optab
, value
,
1305 mask_rtx (mode
, 0, bitsize
, 0),
1306 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
1308 value
= expand_shift (LSHIFT_EXPR
, mode
, value
,
1309 bitnum
, NULL_RTX
, 1);
1313 value
= flip_storage_order (mode
, value
);
1315 /* Now clear the chosen bits in OP0,
1316 except that if VALUE is -1 we need not bother. */
1317 /* We keep the intermediates in registers to allow CSE to combine
1318 consecutive bitfield assignments. */
1320 temp
= force_reg (mode
, op0
);
1324 rtx mask
= mask_rtx (mode
, bitnum
, bitsize
, 1);
1326 mask
= flip_storage_order (mode
, mask
);
1327 temp
= expand_binop (mode
, and_optab
, temp
, mask
,
1328 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
1329 temp
= force_reg (mode
, temp
);
1332 /* Now logical-or VALUE into OP0, unless it is zero. */
1336 temp
= expand_binop (mode
, ior_optab
, temp
, value
,
1337 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
1338 temp
= force_reg (mode
, temp
);
1343 op0
= copy_rtx (op0
);
1344 emit_move_insn (op0
, temp
);
1348 /* Store a bit field that is split across multiple accessible memory objects.
1350 OP0 is the REG, SUBREG or MEM rtx for the first of the objects.
1351 BITSIZE is the field width; BITPOS the position of its first bit
1353 VALUE is the value to store, which has mode VALUE_MODE.
1354 If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
1357 If REVERSE is true, the store is to be done in reverse order.
1359 This does not yet handle fields wider than BITS_PER_WORD. */
1362 store_split_bit_field (rtx op0
, opt_scalar_int_mode op0_mode
,
1363 unsigned HOST_WIDE_INT bitsize
,
1364 unsigned HOST_WIDE_INT bitpos
,
1365 poly_uint64 bitregion_start
, poly_uint64 bitregion_end
,
1366 rtx value
, scalar_int_mode value_mode
, bool reverse
)
1368 unsigned int unit
, total_bits
, bitsdone
= 0;
1370 /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
1372 if (REG_P (op0
) || GET_CODE (op0
) == SUBREG
)
1373 unit
= BITS_PER_WORD
;
1375 unit
= MIN (MEM_ALIGN (op0
), BITS_PER_WORD
);
1377 /* If OP0 is a memory with a mode, then UNIT must not be larger than
1378 OP0's mode as well. Otherwise, store_fixed_bit_field will call us
1379 again, and we will mutually recurse forever. */
1380 if (MEM_P (op0
) && op0_mode
.exists ())
1381 unit
= MIN (unit
, GET_MODE_BITSIZE (op0_mode
.require ()));
1383 /* If VALUE is a constant other than a CONST_INT, get it into a register in
1384 WORD_MODE. If we can do this using gen_lowpart_common, do so. Note
1385 that VALUE might be a floating-point constant. */
1386 if (CONSTANT_P (value
) && !CONST_INT_P (value
))
1388 rtx word
= gen_lowpart_common (word_mode
, value
);
1390 if (word
&& (value
!= word
))
1393 value
= gen_lowpart_common (word_mode
, force_reg (value_mode
, value
));
1394 value_mode
= word_mode
;
1397 total_bits
= GET_MODE_BITSIZE (value_mode
);
1399 while (bitsdone
< bitsize
)
1401 unsigned HOST_WIDE_INT thissize
;
1402 unsigned HOST_WIDE_INT thispos
;
1403 unsigned HOST_WIDE_INT offset
;
1406 offset
= (bitpos
+ bitsdone
) / unit
;
1407 thispos
= (bitpos
+ bitsdone
) % unit
;
1409 /* When region of bytes we can touch is restricted, decrease
1410 UNIT close to the end of the region as needed. If op0 is a REG
1411 or SUBREG of REG, don't do this, as there can't be data races
1412 on a register and we can expand shorter code in some cases. */
1413 if (maybe_ne (bitregion_end
, 0U)
1414 && unit
> BITS_PER_UNIT
1415 && maybe_gt (bitpos
+ bitsdone
- thispos
+ unit
, bitregion_end
+ 1)
1417 && (GET_CODE (op0
) != SUBREG
|| !REG_P (SUBREG_REG (op0
))))
1423 /* THISSIZE must not overrun a word boundary. Otherwise,
1424 store_fixed_bit_field will call us again, and we will mutually
1426 thissize
= MIN (bitsize
- bitsdone
, BITS_PER_WORD
);
1427 thissize
= MIN (thissize
, unit
- thispos
);
1429 if (reverse
? !BYTES_BIG_ENDIAN
: BYTES_BIG_ENDIAN
)
1431 /* Fetch successively less significant portions. */
1432 if (CONST_INT_P (value
))
1433 part
= GEN_INT (((unsigned HOST_WIDE_INT
) (INTVAL (value
))
1434 >> (bitsize
- bitsdone
- thissize
))
1435 & ((HOST_WIDE_INT_1
<< thissize
) - 1));
1436 /* Likewise, but the source is little-endian. */
1438 part
= extract_fixed_bit_field (word_mode
, value
, value_mode
,
1440 bitsize
- bitsdone
- thissize
,
1441 NULL_RTX
, 1, false);
1443 /* The args are chosen so that the last part includes the
1444 lsb. Give extract_bit_field the value it needs (with
1445 endianness compensation) to fetch the piece we want. */
1446 part
= extract_fixed_bit_field (word_mode
, value
, value_mode
,
1448 total_bits
- bitsize
+ bitsdone
,
1449 NULL_RTX
, 1, false);
1453 /* Fetch successively more significant portions. */
1454 if (CONST_INT_P (value
))
1455 part
= GEN_INT (((unsigned HOST_WIDE_INT
) (INTVAL (value
))
1457 & ((HOST_WIDE_INT_1
<< thissize
) - 1));
1458 /* Likewise, but the source is big-endian. */
1460 part
= extract_fixed_bit_field (word_mode
, value
, value_mode
,
1462 total_bits
- bitsdone
- thissize
,
1463 NULL_RTX
, 1, false);
1465 part
= extract_fixed_bit_field (word_mode
, value
, value_mode
,
1466 thissize
, bitsdone
, NULL_RTX
,
1470 /* If OP0 is a register, then handle OFFSET here. */
1471 rtx op0_piece
= op0
;
1472 opt_scalar_int_mode op0_piece_mode
= op0_mode
;
1473 if (SUBREG_P (op0
) || REG_P (op0
))
1475 scalar_int_mode imode
;
1476 if (op0_mode
.exists (&imode
)
1477 && GET_MODE_SIZE (imode
) < UNITS_PER_WORD
)
1480 op0_piece
= const0_rtx
;
1484 op0_piece
= operand_subword_force (op0
,
1485 offset
* unit
/ BITS_PER_WORD
,
1487 op0_piece_mode
= word_mode
;
1489 offset
&= BITS_PER_WORD
/ unit
- 1;
1492 /* OFFSET is in UNITs, and UNIT is in bits. If WORD is const0_rtx,
1493 it is just an out-of-bounds access. Ignore it. */
1494 if (op0_piece
!= const0_rtx
)
1495 store_fixed_bit_field (op0_piece
, op0_piece_mode
, thissize
,
1496 offset
* unit
+ thispos
, bitregion_start
,
1497 bitregion_end
, part
, word_mode
, reverse
);
1498 bitsdone
+= thissize
;
1502 /* A subroutine of extract_bit_field_1 that converts return value X
1503 to either MODE or TMODE. MODE, TMODE and UNSIGNEDP are arguments
1504 to extract_bit_field. */
1507 convert_extracted_bit_field (rtx x
, machine_mode mode
,
1508 machine_mode tmode
, bool unsignedp
)
1510 if (GET_MODE (x
) == tmode
|| GET_MODE (x
) == mode
)
1513 /* If the x mode is not a scalar integral, first convert to the
1514 integer mode of that size and then access it as a floating-point
1515 value via a SUBREG. */
1516 if (!SCALAR_INT_MODE_P (tmode
))
1518 scalar_int_mode int_mode
= int_mode_for_mode (tmode
).require ();
1519 x
= convert_to_mode (int_mode
, x
, unsignedp
);
1520 x
= force_reg (int_mode
, x
);
1521 return gen_lowpart (tmode
, x
);
1524 return convert_to_mode (tmode
, x
, unsignedp
);
1527 /* Try to use an ext(z)v pattern to extract a field from OP0.
1528 Return the extracted value on success, otherwise return null.
1529 EXTV describes the extraction instruction to use. If OP0_MODE
1530 is defined, it is the mode of OP0, otherwise OP0 is a BLKmode MEM.
1531 The other arguments are as for extract_bit_field. */
1534 extract_bit_field_using_extv (const extraction_insn
*extv
, rtx op0
,
1535 opt_scalar_int_mode op0_mode
,
1536 unsigned HOST_WIDE_INT bitsize
,
1537 unsigned HOST_WIDE_INT bitnum
,
1538 int unsignedp
, rtx target
,
1539 machine_mode mode
, machine_mode tmode
)
1541 class expand_operand ops
[4];
1542 rtx spec_target
= target
;
1543 rtx spec_target_subreg
= 0;
1544 scalar_int_mode ext_mode
= extv
->field_mode
;
1545 unsigned unit
= GET_MODE_BITSIZE (ext_mode
);
1547 if (bitsize
== 0 || unit
< bitsize
)
1551 /* Get a reference to the first byte of the field. */
1552 op0
= narrow_bit_field_mem (op0
, extv
->struct_mode
, bitsize
, bitnum
,
1556 /* Convert from counting within OP0 to counting in EXT_MODE. */
1557 if (BYTES_BIG_ENDIAN
)
1558 bitnum
+= unit
- GET_MODE_BITSIZE (op0_mode
.require ());
1560 /* If op0 is a register, we need it in EXT_MODE to make it
1561 acceptable to the format of ext(z)v. */
1562 if (GET_CODE (op0
) == SUBREG
&& op0_mode
.require () != ext_mode
)
1564 if (REG_P (op0
) && op0_mode
.require () != ext_mode
)
1565 op0
= gen_lowpart_SUBREG (ext_mode
, op0
);
1568 /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
1569 "backwards" from the size of the unit we are extracting from.
1570 Otherwise, we count bits from the most significant on a
1571 BYTES/BITS_BIG_ENDIAN machine. */
1573 if (BITS_BIG_ENDIAN
!= BYTES_BIG_ENDIAN
)
1574 bitnum
= unit
- bitsize
- bitnum
;
1577 target
= spec_target
= gen_reg_rtx (tmode
);
1579 if (GET_MODE (target
) != ext_mode
)
1582 /* Don't use LHS paradoxical subreg if explicit truncation is needed
1583 between the mode of the extraction (word_mode) and the target
1584 mode. Instead, create a temporary and use convert_move to set
1587 && TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (target
), ext_mode
)
1588 && (temp
= gen_lowpart_if_possible (ext_mode
, target
)))
1591 if (partial_subreg_p (GET_MODE (spec_target
), ext_mode
))
1592 spec_target_subreg
= target
;
1595 target
= gen_reg_rtx (ext_mode
);
1598 create_output_operand (&ops
[0], target
, ext_mode
);
1599 create_fixed_operand (&ops
[1], op0
);
1600 create_integer_operand (&ops
[2], bitsize
);
1601 create_integer_operand (&ops
[3], bitnum
);
1602 if (maybe_expand_insn (extv
->icode
, 4, ops
))
1604 target
= ops
[0].value
;
1605 if (target
== spec_target
)
1607 if (target
== spec_target_subreg
)
1609 return convert_extracted_bit_field (target
, mode
, tmode
, unsignedp
);
1614 /* See whether it would be valid to extract the part of OP0 with
1615 mode OP0_MODE described by BITNUM and BITSIZE into a value of
1616 mode MODE using a subreg operation.
1617 Return the subreg if so, otherwise return null. */
1620 extract_bit_field_as_subreg (machine_mode mode
, rtx op0
,
1621 machine_mode op0_mode
,
1622 poly_uint64 bitsize
, poly_uint64 bitnum
)
1624 poly_uint64 bytenum
;
1625 if (multiple_p (bitnum
, BITS_PER_UNIT
, &bytenum
)
1626 && known_eq (bitsize
, GET_MODE_BITSIZE (mode
))
1627 && lowpart_bit_field_p (bitnum
, bitsize
, op0_mode
)
1628 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, op0_mode
))
1629 return simplify_gen_subreg (mode
, op0
, op0_mode
, bytenum
);
1633 /* A subroutine of extract_bit_field, with the same arguments.
1634 If FALLBACK_P is true, fall back to extract_fixed_bit_field
1635 if we can find no other means of implementing the operation.
1636 if FALLBACK_P is false, return NULL instead. */
1639 extract_bit_field_1 (rtx str_rtx
, poly_uint64 bitsize
, poly_uint64 bitnum
,
1640 int unsignedp
, rtx target
, machine_mode mode
,
1641 machine_mode tmode
, bool reverse
, bool fallback_p
,
1647 if (tmode
== VOIDmode
)
1650 while (GET_CODE (op0
) == SUBREG
)
1652 bitnum
+= SUBREG_BYTE (op0
) * BITS_PER_UNIT
;
1653 op0
= SUBREG_REG (op0
);
1656 /* If we have an out-of-bounds access to a register, just return an
1657 uninitialized register of the required mode. This can occur if the
1658 source code contains an out-of-bounds access to a small array. */
1659 if (REG_P (op0
) && known_ge (bitnum
, GET_MODE_BITSIZE (GET_MODE (op0
))))
1660 return gen_reg_rtx (tmode
);
1663 && mode
== GET_MODE (op0
)
1664 && known_eq (bitnum
, 0U)
1665 && known_eq (bitsize
, GET_MODE_BITSIZE (GET_MODE (op0
))))
1668 op0
= flip_storage_order (mode
, op0
);
1669 /* We're trying to extract a full register from itself. */
1673 /* First try to check for vector from vector extractions. */
1674 if (VECTOR_MODE_P (GET_MODE (op0
))
1676 && VECTOR_MODE_P (tmode
)
1677 && known_eq (bitsize
, GET_MODE_BITSIZE (tmode
))
1678 && maybe_gt (GET_MODE_SIZE (GET_MODE (op0
)), GET_MODE_SIZE (tmode
)))
1680 machine_mode new_mode
= GET_MODE (op0
);
1681 if (GET_MODE_INNER (new_mode
) != GET_MODE_INNER (tmode
))
1683 scalar_mode inner_mode
= GET_MODE_INNER (tmode
);
1685 if (!multiple_p (GET_MODE_BITSIZE (GET_MODE (op0
)),
1686 GET_MODE_UNIT_BITSIZE (tmode
), &nunits
)
1687 || !related_vector_mode (tmode
, inner_mode
,
1688 nunits
).exists (&new_mode
)
1689 || maybe_ne (GET_MODE_SIZE (new_mode
),
1690 GET_MODE_SIZE (GET_MODE (op0
))))
1691 new_mode
= VOIDmode
;
1694 if (new_mode
!= VOIDmode
1695 && (convert_optab_handler (vec_extract_optab
, new_mode
, tmode
)
1696 != CODE_FOR_nothing
)
1697 && multiple_p (bitnum
, GET_MODE_BITSIZE (tmode
), &pos
))
1699 class expand_operand ops
[3];
1700 machine_mode outermode
= new_mode
;
1701 machine_mode innermode
= tmode
;
1702 enum insn_code icode
1703 = convert_optab_handler (vec_extract_optab
, outermode
, innermode
);
1705 if (new_mode
!= GET_MODE (op0
))
1706 op0
= gen_lowpart (new_mode
, op0
);
1707 create_output_operand (&ops
[0], target
, innermode
);
1709 create_input_operand (&ops
[1], op0
, outermode
);
1710 create_integer_operand (&ops
[2], pos
);
1711 if (maybe_expand_insn (icode
, 3, ops
))
1713 if (alt_rtl
&& ops
[0].target
)
1715 target
= ops
[0].value
;
1716 if (GET_MODE (target
) != mode
)
1717 return gen_lowpart (tmode
, target
);
1723 /* See if we can get a better vector mode before extracting. */
1724 if (VECTOR_MODE_P (GET_MODE (op0
))
1726 && GET_MODE_INNER (GET_MODE (op0
)) != tmode
)
1728 machine_mode new_mode
;
1730 if (GET_MODE_CLASS (tmode
) == MODE_FLOAT
)
1731 new_mode
= MIN_MODE_VECTOR_FLOAT
;
1732 else if (GET_MODE_CLASS (tmode
) == MODE_FRACT
)
1733 new_mode
= MIN_MODE_VECTOR_FRACT
;
1734 else if (GET_MODE_CLASS (tmode
) == MODE_UFRACT
)
1735 new_mode
= MIN_MODE_VECTOR_UFRACT
;
1736 else if (GET_MODE_CLASS (tmode
) == MODE_ACCUM
)
1737 new_mode
= MIN_MODE_VECTOR_ACCUM
;
1738 else if (GET_MODE_CLASS (tmode
) == MODE_UACCUM
)
1739 new_mode
= MIN_MODE_VECTOR_UACCUM
;
1741 new_mode
= MIN_MODE_VECTOR_INT
;
1743 FOR_EACH_MODE_FROM (new_mode
, new_mode
)
1744 if (known_eq (GET_MODE_SIZE (new_mode
), GET_MODE_SIZE (GET_MODE (op0
)))
1745 && known_eq (GET_MODE_UNIT_SIZE (new_mode
), GET_MODE_SIZE (tmode
))
1746 && targetm
.vector_mode_supported_p (new_mode
)
1747 && targetm
.modes_tieable_p (GET_MODE (op0
), new_mode
))
1749 if (new_mode
!= VOIDmode
)
1750 op0
= gen_lowpart (new_mode
, op0
);
1753 /* Use vec_extract patterns for extracting parts of vectors whenever
1754 available. If that fails, see whether the current modes and bitregion
1755 give a natural subreg. */
1756 machine_mode outermode
= GET_MODE (op0
);
1757 if (VECTOR_MODE_P (outermode
) && !MEM_P (op0
))
1759 scalar_mode innermode
= GET_MODE_INNER (outermode
);
1760 enum insn_code icode
1761 = convert_optab_handler (vec_extract_optab
, outermode
, innermode
);
1763 if (icode
!= CODE_FOR_nothing
1764 && known_eq (bitsize
, GET_MODE_BITSIZE (innermode
))
1765 && multiple_p (bitnum
, GET_MODE_BITSIZE (innermode
), &pos
))
1767 class expand_operand ops
[3];
1769 create_output_operand (&ops
[0], target
, innermode
);
1771 create_input_operand (&ops
[1], op0
, outermode
);
1772 create_integer_operand (&ops
[2], pos
);
1773 if (maybe_expand_insn (icode
, 3, ops
))
1775 if (alt_rtl
&& ops
[0].target
)
1777 target
= ops
[0].value
;
1778 if (GET_MODE (target
) != mode
)
1779 return gen_lowpart (tmode
, target
);
1783 /* Using subregs is useful if we're extracting one register vector
1784 from a multi-register vector. extract_bit_field_as_subreg checks
1785 for valid bitsize and bitnum, so we don't need to do that here. */
1786 if (VECTOR_MODE_P (mode
))
1788 rtx sub
= extract_bit_field_as_subreg (mode
, op0
, outermode
,
1795 /* Make sure we are playing with integral modes. Pun with subregs
1797 opt_scalar_int_mode op0_mode
= int_mode_for_mode (GET_MODE (op0
));
1798 scalar_int_mode imode
;
1799 if (!op0_mode
.exists (&imode
) || imode
!= GET_MODE (op0
))
1802 op0
= adjust_bitfield_address_size (op0
, op0_mode
.else_blk (),
1804 else if (op0_mode
.exists (&imode
))
1806 op0
= gen_lowpart (imode
, op0
);
1808 /* If we got a SUBREG, force it into a register since we
1809 aren't going to be able to do another SUBREG on it. */
1810 if (GET_CODE (op0
) == SUBREG
)
1811 op0
= force_reg (imode
, op0
);
1815 poly_int64 size
= GET_MODE_SIZE (GET_MODE (op0
));
1816 rtx mem
= assign_stack_temp (GET_MODE (op0
), size
);
1817 emit_move_insn (mem
, op0
);
1818 op0
= adjust_bitfield_address_size (mem
, BLKmode
, 0, size
);
1822 /* ??? We currently assume TARGET is at least as big as BITSIZE.
1823 If that's wrong, the solution is to test for it and set TARGET to 0
1826 /* Get the mode of the field to use for atomic access or subreg
1828 if (!SCALAR_INT_MODE_P (tmode
)
1829 || !mode_for_size (bitsize
, GET_MODE_CLASS (tmode
), 0).exists (&mode1
))
1831 gcc_assert (mode1
!= BLKmode
);
1833 /* Extraction of a full MODE1 value can be done with a subreg as long
1834 as the least significant bit of the value is the least significant
1835 bit of either OP0 or a word of OP0. */
1836 if (!MEM_P (op0
) && !reverse
&& op0_mode
.exists (&imode
))
1838 rtx sub
= extract_bit_field_as_subreg (mode1
, op0
, imode
,
1841 return convert_extracted_bit_field (sub
, mode
, tmode
, unsignedp
);
1844 /* Extraction of a full MODE1 value can be done with a load as long as
1845 the field is on a byte boundary and is sufficiently aligned. */
1846 poly_uint64 bytenum
;
1847 if (simple_mem_bitfield_p (op0
, bitsize
, bitnum
, mode1
, &bytenum
))
1849 op0
= adjust_bitfield_address (op0
, mode1
, bytenum
);
1851 op0
= flip_storage_order (mode1
, op0
);
1852 return convert_extracted_bit_field (op0
, mode
, tmode
, unsignedp
);
1855 /* If we have a memory source and a non-constant bit offset, restrict
1856 the memory to the referenced bytes. This is a worst-case fallback
1857 but is useful for things like vector booleans. */
1858 if (MEM_P (op0
) && !bitnum
.is_constant ())
1860 bytenum
= bits_to_bytes_round_down (bitnum
);
1861 bitnum
= num_trailing_bits (bitnum
);
1862 poly_uint64 bytesize
= bits_to_bytes_round_up (bitnum
+ bitsize
);
1863 op0
= adjust_bitfield_address_size (op0
, BLKmode
, bytenum
, bytesize
);
1864 op0_mode
= opt_scalar_int_mode ();
1867 /* It's possible we'll need to handle other cases here for
1868 polynomial bitnum and bitsize. */
1870 /* From here on we need to be looking at a fixed-size insertion. */
1871 return extract_integral_bit_field (op0
, op0_mode
, bitsize
.to_constant (),
1872 bitnum
.to_constant (), unsignedp
,
1873 target
, mode
, tmode
, reverse
, fallback_p
);
1876 /* Subroutine of extract_bit_field_1, with the same arguments, except
1877 that BITSIZE and BITNUM are constant. Handle cases specific to
1878 integral modes. If OP0_MODE is defined, it is the mode of OP0,
1879 otherwise OP0 is a BLKmode MEM. */
1882 extract_integral_bit_field (rtx op0
, opt_scalar_int_mode op0_mode
,
1883 unsigned HOST_WIDE_INT bitsize
,
1884 unsigned HOST_WIDE_INT bitnum
, int unsignedp
,
1885 rtx target
, machine_mode mode
, machine_mode tmode
,
1886 bool reverse
, bool fallback_p
)
1888 /* Handle fields bigger than a word. */
1890 if (bitsize
> BITS_PER_WORD
)
1892 /* Here we transfer the words of the field
1893 in the order least significant first.
1894 This is because the most significant word is the one which may
1895 be less than full. */
1897 const bool backwards
= WORDS_BIG_ENDIAN
;
1898 unsigned int nwords
= (bitsize
+ (BITS_PER_WORD
- 1)) / BITS_PER_WORD
;
1902 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1903 target
= gen_reg_rtx (mode
);
1905 /* In case we're about to clobber a base register or something
1906 (see gcc.c-torture/execute/20040625-1.c). */
1907 if (reg_mentioned_p (target
, op0
))
1908 target
= gen_reg_rtx (mode
);
1910 /* Indicate for flow that the entire target reg is being set. */
1911 emit_clobber (target
);
1913 /* The mode must be fixed-size, since extract_bit_field_1 handles
1914 extractions from variable-sized objects before calling this
1916 unsigned int target_size
1917 = GET_MODE_SIZE (GET_MODE (target
)).to_constant ();
1918 last
= get_last_insn ();
1919 for (i
= 0; i
< nwords
; i
++)
1921 /* If I is 0, use the low-order word in both field and target;
1922 if I is 1, use the next to lowest word; and so on. */
1923 /* Word number in TARGET to use. */
1924 unsigned int wordnum
1925 = (backwards
? target_size
/ UNITS_PER_WORD
- i
- 1 : i
);
1926 /* Offset from start of field in OP0. */
1927 unsigned int bit_offset
= (backwards
^ reverse
1928 ? MAX ((int) bitsize
- ((int) i
+ 1)
1931 : (int) i
* BITS_PER_WORD
);
1932 rtx target_part
= operand_subword (target
, wordnum
, 1, VOIDmode
);
1934 = extract_bit_field_1 (op0
, MIN (BITS_PER_WORD
,
1935 bitsize
- i
* BITS_PER_WORD
),
1936 bitnum
+ bit_offset
, 1, target_part
,
1937 mode
, word_mode
, reverse
, fallback_p
, NULL
);
1939 gcc_assert (target_part
);
1942 delete_insns_since (last
);
1946 if (result_part
!= target_part
)
1947 emit_move_insn (target_part
, result_part
);
1952 /* Unless we've filled TARGET, the upper regs in a multi-reg value
1953 need to be zero'd out. */
1954 if (target_size
> nwords
* UNITS_PER_WORD
)
1956 unsigned int i
, total_words
;
1958 total_words
= target_size
/ UNITS_PER_WORD
;
1959 for (i
= nwords
; i
< total_words
; i
++)
1961 (operand_subword (target
,
1962 backwards
? total_words
- i
- 1 : i
,
1969 /* Signed bit field: sign-extend with two arithmetic shifts. */
1970 target
= expand_shift (LSHIFT_EXPR
, mode
, target
,
1971 GET_MODE_BITSIZE (mode
) - bitsize
, NULL_RTX
, 0);
1972 return expand_shift (RSHIFT_EXPR
, mode
, target
,
1973 GET_MODE_BITSIZE (mode
) - bitsize
, NULL_RTX
, 0);
1976 /* If OP0 is a multi-word register, narrow it to the affected word.
1977 If the region spans two words, defer to extract_split_bit_field. */
1978 if (!MEM_P (op0
) && GET_MODE_SIZE (op0_mode
.require ()) > UNITS_PER_WORD
)
1980 if (bitnum
% BITS_PER_WORD
+ bitsize
> BITS_PER_WORD
)
1984 target
= extract_split_bit_field (op0
, op0_mode
, bitsize
, bitnum
,
1985 unsignedp
, reverse
);
1986 return convert_extracted_bit_field (target
, mode
, tmode
, unsignedp
);
1988 /* If OP0 is a hard register, copy it to a pseudo before calling
1989 simplify_gen_subreg. */
1990 if (REG_P (op0
) && HARD_REGISTER_P (op0
))
1991 op0
= copy_to_reg (op0
);
1992 op0
= simplify_gen_subreg (word_mode
, op0
, op0_mode
.require (),
1993 bitnum
/ BITS_PER_WORD
* UNITS_PER_WORD
);
1994 op0_mode
= word_mode
;
1995 bitnum
%= BITS_PER_WORD
;
1998 /* From here on we know the desired field is smaller than a word.
1999 If OP0 is a register, it too fits within a word. */
2000 enum extraction_pattern pattern
= unsignedp
? EP_extzv
: EP_extv
;
2001 extraction_insn extv
;
2004 /* ??? We could limit the structure size to the part of OP0 that
2005 contains the field, with appropriate checks for endianness
2006 and TARGET_TRULY_NOOP_TRUNCATION. */
2007 && get_best_reg_extraction_insn (&extv
, pattern
,
2008 GET_MODE_BITSIZE (op0_mode
.require ()),
2011 rtx result
= extract_bit_field_using_extv (&extv
, op0
, op0_mode
,
2013 unsignedp
, target
, mode
,
2019 /* If OP0 is a memory, try copying it to a register and seeing if a
2020 cheap register alternative is available. */
2021 if (MEM_P (op0
) & !reverse
)
2023 if (get_best_mem_extraction_insn (&extv
, pattern
, bitsize
, bitnum
,
2026 rtx result
= extract_bit_field_using_extv (&extv
, op0
, op0_mode
,
2028 unsignedp
, target
, mode
,
2034 rtx_insn
*last
= get_last_insn ();
2036 /* Try loading part of OP0 into a register and extracting the
2037 bitfield from that. */
2038 unsigned HOST_WIDE_INT bitpos
;
2039 rtx xop0
= adjust_bit_field_mem_for_reg (pattern
, op0
, bitsize
, bitnum
,
2040 0, 0, tmode
, &bitpos
);
2043 xop0
= copy_to_reg (xop0
);
2044 rtx result
= extract_bit_field_1 (xop0
, bitsize
, bitpos
,
2046 mode
, tmode
, reverse
, false, NULL
);
2049 delete_insns_since (last
);
2056 /* Find a correspondingly-sized integer field, so we can apply
2057 shifts and masks to it. */
2058 scalar_int_mode int_mode
;
2059 if (!int_mode_for_mode (tmode
).exists (&int_mode
))
2060 /* If this fails, we should probably push op0 out to memory and then
2062 int_mode
= int_mode_for_mode (mode
).require ();
2064 target
= extract_fixed_bit_field (int_mode
, op0
, op0_mode
, bitsize
,
2065 bitnum
, target
, unsignedp
, reverse
);
2067 /* Complex values must be reversed piecewise, so we need to undo the global
2068 reversal, convert to the complex mode and reverse again. */
2069 if (reverse
&& COMPLEX_MODE_P (tmode
))
2071 target
= flip_storage_order (int_mode
, target
);
2072 target
= convert_extracted_bit_field (target
, mode
, tmode
, unsignedp
);
2073 target
= flip_storage_order (tmode
, target
);
2076 target
= convert_extracted_bit_field (target
, mode
, tmode
, unsignedp
);
2081 /* Generate code to extract a byte-field from STR_RTX
2082 containing BITSIZE bits, starting at BITNUM,
2083 and put it in TARGET if possible (if TARGET is nonzero).
2084 Regardless of TARGET, we return the rtx for where the value is placed.
2086 STR_RTX is the structure containing the byte (a REG or MEM).
2087 UNSIGNEDP is nonzero if this is an unsigned bit field.
2088 MODE is the natural mode of the field value once extracted.
2089 TMODE is the mode the caller would like the value to have;
2090 but the value may be returned with type MODE instead.
2092 If REVERSE is true, the extraction is to be done in reverse order.
2094 If a TARGET is specified and we can store in it at no extra cost,
2095 we do so, and return TARGET.
2096 Otherwise, we return a REG of mode TMODE or MODE, with TMODE preferred
2097 if they are equally easy.
2099 If the result can be stored at TARGET, and ALT_RTL is non-NULL,
2100 then *ALT_RTL is set to TARGET (before legitimziation). */
2103 extract_bit_field (rtx str_rtx
, poly_uint64 bitsize
, poly_uint64 bitnum
,
2104 int unsignedp
, rtx target
, machine_mode mode
,
2105 machine_mode tmode
, bool reverse
, rtx
*alt_rtl
)
2109 /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
2110 if (maybe_ne (GET_MODE_BITSIZE (GET_MODE (str_rtx
)), 0))
2111 mode1
= GET_MODE (str_rtx
);
2112 else if (target
&& maybe_ne (GET_MODE_BITSIZE (GET_MODE (target
)), 0))
2113 mode1
= GET_MODE (target
);
2117 unsigned HOST_WIDE_INT ibitsize
, ibitnum
;
2118 scalar_int_mode int_mode
;
2119 if (bitsize
.is_constant (&ibitsize
)
2120 && bitnum
.is_constant (&ibitnum
)
2121 && is_a
<scalar_int_mode
> (mode1
, &int_mode
)
2122 && strict_volatile_bitfield_p (str_rtx
, ibitsize
, ibitnum
,
2125 /* Extraction of a full INT_MODE value can be done with a simple load.
2126 We know here that the field can be accessed with one single
2127 instruction. For targets that support unaligned memory,
2128 an unaligned access may be necessary. */
2129 if (ibitsize
== GET_MODE_BITSIZE (int_mode
))
2131 rtx result
= adjust_bitfield_address (str_rtx
, int_mode
,
2132 ibitnum
/ BITS_PER_UNIT
);
2134 result
= flip_storage_order (int_mode
, result
);
2135 gcc_assert (ibitnum
% BITS_PER_UNIT
== 0);
2136 return convert_extracted_bit_field (result
, mode
, tmode
, unsignedp
);
2139 str_rtx
= narrow_bit_field_mem (str_rtx
, int_mode
, ibitsize
, ibitnum
,
2141 gcc_assert (ibitnum
+ ibitsize
<= GET_MODE_BITSIZE (int_mode
));
2142 str_rtx
= copy_to_reg (str_rtx
);
2143 return extract_bit_field_1 (str_rtx
, ibitsize
, ibitnum
, unsignedp
,
2144 target
, mode
, tmode
, reverse
, true, alt_rtl
);
2147 return extract_bit_field_1 (str_rtx
, bitsize
, bitnum
, unsignedp
,
2148 target
, mode
, tmode
, reverse
, true, alt_rtl
);
2151 /* Use shifts and boolean operations to extract a field of BITSIZE bits
2152 from bit BITNUM of OP0. If OP0_MODE is defined, it is the mode of OP0,
2153 otherwise OP0 is a BLKmode MEM.
2155 UNSIGNEDP is nonzero for an unsigned bit field (don't sign-extend value).
2156 If REVERSE is true, the extraction is to be done in reverse order.
2158 If TARGET is nonzero, attempts to store the value there
2159 and return TARGET, but this is not guaranteed.
2160 If TARGET is not used, create a pseudo-reg of mode TMODE for the value. */
2163 extract_fixed_bit_field (machine_mode tmode
, rtx op0
,
2164 opt_scalar_int_mode op0_mode
,
2165 unsigned HOST_WIDE_INT bitsize
,
2166 unsigned HOST_WIDE_INT bitnum
, rtx target
,
2167 int unsignedp
, bool reverse
)
2169 scalar_int_mode mode
;
2172 if (!get_best_mode (bitsize
, bitnum
, 0, 0, MEM_ALIGN (op0
),
2173 BITS_PER_WORD
, MEM_VOLATILE_P (op0
), &mode
))
2174 /* The only way this should occur is if the field spans word
2176 return extract_split_bit_field (op0
, op0_mode
, bitsize
, bitnum
,
2177 unsignedp
, reverse
);
2179 op0
= narrow_bit_field_mem (op0
, mode
, bitsize
, bitnum
, &bitnum
);
2182 mode
= op0_mode
.require ();
2184 return extract_fixed_bit_field_1 (tmode
, op0
, mode
, bitsize
, bitnum
,
2185 target
, unsignedp
, reverse
);
2188 /* Helper function for extract_fixed_bit_field, extracts
2189 the bit field always using MODE, which is the mode of OP0.
2190 The other arguments are as for extract_fixed_bit_field. */
2193 extract_fixed_bit_field_1 (machine_mode tmode
, rtx op0
, scalar_int_mode mode
,
2194 unsigned HOST_WIDE_INT bitsize
,
2195 unsigned HOST_WIDE_INT bitnum
, rtx target
,
2196 int unsignedp
, bool reverse
)
2198 /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
2199 for invalid input, such as extract equivalent of f5 from
2200 gcc.dg/pr48335-2.c. */
2202 if (reverse
? !BYTES_BIG_ENDIAN
: BYTES_BIG_ENDIAN
)
2203 /* BITNUM is the distance between our msb and that of OP0.
2204 Convert it to the distance from the lsb. */
2205 bitnum
= GET_MODE_BITSIZE (mode
) - bitsize
- bitnum
;
2207 /* Now BITNUM is always the distance between the field's lsb and that of OP0.
2208 We have reduced the big-endian case to the little-endian case. */
2210 op0
= flip_storage_order (mode
, op0
);
2216 /* If the field does not already start at the lsb,
2217 shift it so it does. */
2218 /* Maybe propagate the target for the shift. */
2219 rtx subtarget
= (target
!= 0 && REG_P (target
) ? target
: 0);
2222 op0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, bitnum
, subtarget
, 1);
2224 /* Convert the value to the desired mode. TMODE must also be a
2225 scalar integer for this conversion to make sense, since we
2226 shouldn't reinterpret the bits. */
2227 scalar_int_mode new_mode
= as_a
<scalar_int_mode
> (tmode
);
2228 if (mode
!= new_mode
)
2229 op0
= convert_to_mode (new_mode
, op0
, 1);
2231 /* Unless the msb of the field used to be the msb when we shifted,
2232 mask out the upper bits. */
2234 if (GET_MODE_BITSIZE (mode
) != bitnum
+ bitsize
)
2235 return expand_binop (new_mode
, and_optab
, op0
,
2236 mask_rtx (new_mode
, 0, bitsize
, 0),
2237 target
, 1, OPTAB_LIB_WIDEN
);
2241 /* To extract a signed bit-field, first shift its msb to the msb of the word,
2242 then arithmetic-shift its lsb to the lsb of the word. */
2243 op0
= force_reg (mode
, op0
);
2245 /* Find the narrowest integer mode that contains the field. */
2247 opt_scalar_int_mode mode_iter
;
2248 FOR_EACH_MODE_IN_CLASS (mode_iter
, MODE_INT
)
2249 if (GET_MODE_BITSIZE (mode_iter
.require ()) >= bitsize
+ bitnum
)
2252 mode
= mode_iter
.require ();
2253 op0
= convert_to_mode (mode
, op0
, 0);
2258 if (GET_MODE_BITSIZE (mode
) != (bitsize
+ bitnum
))
2260 int amount
= GET_MODE_BITSIZE (mode
) - (bitsize
+ bitnum
);
2261 /* Maybe propagate the target for the shift. */
2262 rtx subtarget
= (target
!= 0 && REG_P (target
) ? target
: 0);
2263 op0
= expand_shift (LSHIFT_EXPR
, mode
, op0
, amount
, subtarget
, 1);
2266 return expand_shift (RSHIFT_EXPR
, mode
, op0
,
2267 GET_MODE_BITSIZE (mode
) - bitsize
, target
, 0);
2270 /* Return a constant integer (CONST_INT or CONST_DOUBLE) rtx with the value
2274 lshift_value (machine_mode mode
, unsigned HOST_WIDE_INT value
,
2277 return immed_wide_int_const (wi::lshift (value
, bitpos
), mode
);
2280 /* Extract a bit field that is split across two words
2281 and return an RTX for the result.
2283 OP0 is the REG, SUBREG or MEM rtx for the first of the two words.
2284 BITSIZE is the field width; BITPOS, position of its first bit, in the word.
2285 UNSIGNEDP is 1 if should zero-extend the contents; else sign-extend.
2286 If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
2289 If REVERSE is true, the extraction is to be done in reverse order. */
2292 extract_split_bit_field (rtx op0
, opt_scalar_int_mode op0_mode
,
2293 unsigned HOST_WIDE_INT bitsize
,
2294 unsigned HOST_WIDE_INT bitpos
, int unsignedp
,
2298 unsigned int bitsdone
= 0;
2299 rtx result
= NULL_RTX
;
2302 /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
2304 if (REG_P (op0
) || GET_CODE (op0
) == SUBREG
)
2305 unit
= BITS_PER_WORD
;
2307 unit
= MIN (MEM_ALIGN (op0
), BITS_PER_WORD
);
2309 while (bitsdone
< bitsize
)
2311 unsigned HOST_WIDE_INT thissize
;
2313 unsigned HOST_WIDE_INT thispos
;
2314 unsigned HOST_WIDE_INT offset
;
2316 offset
= (bitpos
+ bitsdone
) / unit
;
2317 thispos
= (bitpos
+ bitsdone
) % unit
;
2319 /* THISSIZE must not overrun a word boundary. Otherwise,
2320 extract_fixed_bit_field will call us again, and we will mutually
2322 thissize
= MIN (bitsize
- bitsdone
, BITS_PER_WORD
);
2323 thissize
= MIN (thissize
, unit
- thispos
);
2325 /* If OP0 is a register, then handle OFFSET here. */
2326 rtx op0_piece
= op0
;
2327 opt_scalar_int_mode op0_piece_mode
= op0_mode
;
2328 if (SUBREG_P (op0
) || REG_P (op0
))
2330 op0_piece
= operand_subword_force (op0
, offset
, op0_mode
.require ());
2331 op0_piece_mode
= word_mode
;
2335 /* Extract the parts in bit-counting order,
2336 whose meaning is determined by BYTES_PER_UNIT.
2337 OFFSET is in UNITs, and UNIT is in bits. */
2338 part
= extract_fixed_bit_field (word_mode
, op0_piece
, op0_piece_mode
,
2339 thissize
, offset
* unit
+ thispos
,
2341 bitsdone
+= thissize
;
2343 /* Shift this part into place for the result. */
2344 if (reverse
? !BYTES_BIG_ENDIAN
: BYTES_BIG_ENDIAN
)
2346 if (bitsize
!= bitsdone
)
2347 part
= expand_shift (LSHIFT_EXPR
, word_mode
, part
,
2348 bitsize
- bitsdone
, 0, 1);
2352 if (bitsdone
!= thissize
)
2353 part
= expand_shift (LSHIFT_EXPR
, word_mode
, part
,
2354 bitsdone
- thissize
, 0, 1);
2360 /* Combine the parts with bitwise or. This works
2361 because we extracted each part as an unsigned bit field. */
2362 result
= expand_binop (word_mode
, ior_optab
, part
, result
, NULL_RTX
, 1,
2368 /* Unsigned bit field: we are done. */
2371 /* Signed bit field: sign-extend with two arithmetic shifts. */
2372 result
= expand_shift (LSHIFT_EXPR
, word_mode
, result
,
2373 BITS_PER_WORD
- bitsize
, NULL_RTX
, 0);
2374 return expand_shift (RSHIFT_EXPR
, word_mode
, result
,
2375 BITS_PER_WORD
- bitsize
, NULL_RTX
, 0);
2378 /* Try to read the low bits of SRC as an rvalue of mode MODE, preserving
2379 the bit pattern. SRC_MODE is the mode of SRC; if this is smaller than
2380 MODE, fill the upper bits with zeros. Fail if the layout of either
2381 mode is unknown (as for CC modes) or if the extraction would involve
2382 unprofitable mode punning. Return the value on success, otherwise
2385 This is different from gen_lowpart* in these respects:
2387 - the returned value must always be considered an rvalue
2389 - when MODE is wider than SRC_MODE, the extraction involves
2392 - when MODE is smaller than SRC_MODE, the extraction involves
2393 a truncation (and is thus subject to TARGET_TRULY_NOOP_TRUNCATION).
2395 In other words, this routine performs a computation, whereas the
2396 gen_lowpart* routines are conceptually lvalue or rvalue subreg
2400 extract_low_bits (machine_mode mode
, machine_mode src_mode
, rtx src
)
2402 scalar_int_mode int_mode
, src_int_mode
;
2404 if (mode
== src_mode
)
2407 if (CONSTANT_P (src
))
2409 /* simplify_gen_subreg can't be used here, as if simplify_subreg
2410 fails, it will happily create (subreg (symbol_ref)) or similar
2412 poly_uint64 byte
= subreg_lowpart_offset (mode
, src_mode
);
2413 rtx ret
= simplify_subreg (mode
, src
, src_mode
, byte
);
2417 if (GET_MODE (src
) == VOIDmode
2418 || !validate_subreg (mode
, src_mode
, src
, byte
))
2421 src
= force_reg (GET_MODE (src
), src
);
2422 return gen_rtx_SUBREG (mode
, src
, byte
);
2425 if (GET_MODE_CLASS (mode
) == MODE_CC
|| GET_MODE_CLASS (src_mode
) == MODE_CC
)
2428 if (known_eq (GET_MODE_BITSIZE (mode
), GET_MODE_BITSIZE (src_mode
))
2429 && targetm
.modes_tieable_p (mode
, src_mode
))
2431 rtx x
= gen_lowpart_common (mode
, src
);
2436 if (!int_mode_for_mode (src_mode
).exists (&src_int_mode
)
2437 || !int_mode_for_mode (mode
).exists (&int_mode
))
2440 if (!targetm
.modes_tieable_p (src_int_mode
, src_mode
))
2442 if (!targetm
.modes_tieable_p (int_mode
, mode
))
2445 src
= gen_lowpart (src_int_mode
, src
);
2446 if (!validate_subreg (int_mode
, src_int_mode
, src
,
2447 subreg_lowpart_offset (int_mode
, src_int_mode
)))
2450 src
= convert_modes (int_mode
, src_int_mode
, src
, true);
2451 src
= gen_lowpart (mode
, src
);
2455 /* Add INC into TARGET. */
2458 expand_inc (rtx target
, rtx inc
)
2460 rtx value
= expand_binop (GET_MODE (target
), add_optab
,
2462 target
, 0, OPTAB_LIB_WIDEN
);
2463 if (value
!= target
)
2464 emit_move_insn (target
, value
);
2467 /* Subtract DEC from TARGET. */
2470 expand_dec (rtx target
, rtx dec
)
2472 rtx value
= expand_binop (GET_MODE (target
), sub_optab
,
2474 target
, 0, OPTAB_LIB_WIDEN
);
2475 if (value
!= target
)
2476 emit_move_insn (target
, value
);
2479 /* Output a shift instruction for expression code CODE,
2480 with SHIFTED being the rtx for the value to shift,
2481 and AMOUNT the rtx for the amount to shift by.
2482 Store the result in the rtx TARGET, if that is convenient.
2483 If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2484 Return the rtx for where the value is.
2485 If that cannot be done, abort the compilation unless MAY_FAIL is true,
2486 in which case 0 is returned. */
2489 expand_shift_1 (enum tree_code code
, machine_mode mode
, rtx shifted
,
2490 rtx amount
, rtx target
, int unsignedp
, bool may_fail
= false)
2493 int left
= (code
== LSHIFT_EXPR
|| code
== LROTATE_EXPR
);
2494 int rotate
= (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
);
2495 optab lshift_optab
= ashl_optab
;
2496 optab rshift_arith_optab
= ashr_optab
;
2497 optab rshift_uns_optab
= lshr_optab
;
2498 optab lrotate_optab
= rotl_optab
;
2499 optab rrotate_optab
= rotr_optab
;
2500 machine_mode op1_mode
;
2501 scalar_mode scalar_mode
= GET_MODE_INNER (mode
);
2503 bool speed
= optimize_insn_for_speed_p ();
2506 op1_mode
= GET_MODE (op1
);
2508 /* Determine whether the shift/rotate amount is a vector, or scalar. If the
2509 shift amount is a vector, use the vector/vector shift patterns. */
2510 if (VECTOR_MODE_P (mode
) && VECTOR_MODE_P (op1_mode
))
2512 lshift_optab
= vashl_optab
;
2513 rshift_arith_optab
= vashr_optab
;
2514 rshift_uns_optab
= vlshr_optab
;
2515 lrotate_optab
= vrotl_optab
;
2516 rrotate_optab
= vrotr_optab
;
2519 /* Previously detected shift-counts computed by NEGATE_EXPR
2520 and shifted in the other direction; but that does not work
2523 if (SHIFT_COUNT_TRUNCATED
)
2525 if (CONST_INT_P (op1
)
2526 && ((unsigned HOST_WIDE_INT
) INTVAL (op1
) >=
2527 (unsigned HOST_WIDE_INT
) GET_MODE_BITSIZE (scalar_mode
)))
2528 op1
= gen_int_shift_amount (mode
,
2529 (unsigned HOST_WIDE_INT
) INTVAL (op1
)
2530 % GET_MODE_BITSIZE (scalar_mode
));
2531 else if (GET_CODE (op1
) == SUBREG
2532 && subreg_lowpart_p (op1
)
2533 && SCALAR_INT_MODE_P (GET_MODE (SUBREG_REG (op1
)))
2534 && SCALAR_INT_MODE_P (GET_MODE (op1
)))
2535 op1
= SUBREG_REG (op1
);
2538 /* Canonicalize rotates by constant amount. If op1 is bitsize / 2,
2539 prefer left rotation, if op1 is from bitsize / 2 + 1 to
2540 bitsize - 1, use other direction of rotate with 1 .. bitsize / 2 - 1
2543 && CONST_INT_P (op1
)
2544 && IN_RANGE (INTVAL (op1
), GET_MODE_BITSIZE (scalar_mode
) / 2 + left
,
2545 GET_MODE_BITSIZE (scalar_mode
) - 1))
2547 op1
= gen_int_shift_amount (mode
, (GET_MODE_BITSIZE (scalar_mode
)
2550 code
= left
? LROTATE_EXPR
: RROTATE_EXPR
;
2553 /* Rotation of 16bit values by 8 bits is effectively equivalent to a bswaphi.
2554 Note that this is not the case for bigger values. For instance a rotation
2555 of 0x01020304 by 16 bits gives 0x03040102 which is different from
2556 0x04030201 (bswapsi). */
2558 && CONST_INT_P (op1
)
2559 && INTVAL (op1
) == BITS_PER_UNIT
2560 && GET_MODE_SIZE (scalar_mode
) == 2
2561 && optab_handler (bswap_optab
, mode
) != CODE_FOR_nothing
)
2562 return expand_unop (mode
, bswap_optab
, shifted
, NULL_RTX
, unsignedp
);
2564 if (op1
== const0_rtx
)
2567 /* Check whether its cheaper to implement a left shift by a constant
2568 bit count by a sequence of additions. */
2569 if (code
== LSHIFT_EXPR
2570 && CONST_INT_P (op1
)
2572 && INTVAL (op1
) < GET_MODE_PRECISION (scalar_mode
)
2573 && INTVAL (op1
) < MAX_BITS_PER_WORD
2574 && (shift_cost (speed
, mode
, INTVAL (op1
))
2575 > INTVAL (op1
) * add_cost (speed
, mode
))
2576 && shift_cost (speed
, mode
, INTVAL (op1
)) != MAX_COST
)
2579 for (i
= 0; i
< INTVAL (op1
); i
++)
2581 temp
= force_reg (mode
, shifted
);
2582 shifted
= expand_binop (mode
, add_optab
, temp
, temp
, NULL_RTX
,
2583 unsignedp
, OPTAB_LIB_WIDEN
);
2588 for (attempt
= 0; temp
== 0 && attempt
< 3; attempt
++)
2590 enum optab_methods methods
;
2593 methods
= OPTAB_DIRECT
;
2594 else if (attempt
== 1)
2595 methods
= OPTAB_WIDEN
;
2597 methods
= OPTAB_LIB_WIDEN
;
2601 /* Widening does not work for rotation. */
2602 if (methods
== OPTAB_WIDEN
)
2604 else if (methods
== OPTAB_LIB_WIDEN
)
2606 /* If we have been unable to open-code this by a rotation,
2607 do it as the IOR of two shifts. I.e., to rotate A
2609 (A << N) | ((unsigned) A >> ((-N) & (C - 1)))
2610 where C is the bitsize of A.
2612 It is theoretically possible that the target machine might
2613 not be able to perform either shift and hence we would
2614 be making two libcalls rather than just the one for the
2615 shift (similarly if IOR could not be done). We will allow
2616 this extremely unlikely lossage to avoid complicating the
2619 rtx subtarget
= target
== shifted
? 0 : target
;
2620 rtx new_amount
, other_amount
;
2624 if (op1
== const0_rtx
)
2626 else if (CONST_INT_P (op1
))
2627 other_amount
= gen_int_shift_amount
2628 (mode
, GET_MODE_BITSIZE (scalar_mode
) - INTVAL (op1
));
2632 = simplify_gen_unary (NEG
, GET_MODE (op1
),
2633 op1
, GET_MODE (op1
));
2634 HOST_WIDE_INT mask
= GET_MODE_PRECISION (scalar_mode
) - 1;
2636 = simplify_gen_binary (AND
, GET_MODE (op1
), other_amount
,
2637 gen_int_mode (mask
, GET_MODE (op1
)));
2640 shifted
= force_reg (mode
, shifted
);
2642 temp
= expand_shift_1 (left
? LSHIFT_EXPR
: RSHIFT_EXPR
,
2643 mode
, shifted
, new_amount
, 0, 1);
2644 temp1
= expand_shift_1 (left
? RSHIFT_EXPR
: LSHIFT_EXPR
,
2645 mode
, shifted
, other_amount
,
2647 return expand_binop (mode
, ior_optab
, temp
, temp1
, target
,
2648 unsignedp
, methods
);
2651 temp
= expand_binop (mode
,
2652 left
? lrotate_optab
: rrotate_optab
,
2653 shifted
, op1
, target
, unsignedp
, methods
);
2656 temp
= expand_binop (mode
,
2657 left
? lshift_optab
: rshift_uns_optab
,
2658 shifted
, op1
, target
, unsignedp
, methods
);
2660 /* Do arithmetic shifts.
2661 Also, if we are going to widen the operand, we can just as well
2662 use an arithmetic right-shift instead of a logical one. */
2663 if (temp
== 0 && ! rotate
2664 && (! unsignedp
|| (! left
&& methods
== OPTAB_WIDEN
)))
2666 enum optab_methods methods1
= methods
;
2668 /* If trying to widen a log shift to an arithmetic shift,
2669 don't accept an arithmetic shift of the same size. */
2671 methods1
= OPTAB_MUST_WIDEN
;
2673 /* Arithmetic shift */
2675 temp
= expand_binop (mode
,
2676 left
? lshift_optab
: rshift_arith_optab
,
2677 shifted
, op1
, target
, unsignedp
, methods1
);
2680 /* We used to try extzv here for logical right shifts, but that was
2681 only useful for one machine, the VAX, and caused poor code
2682 generation there for lshrdi3, so the code was deleted and a
2683 define_expand for lshrsi3 was added to vax.md. */
2686 gcc_assert (temp
!= NULL_RTX
|| may_fail
);
2690 /* Output a shift instruction for expression code CODE,
2691 with SHIFTED being the rtx for the value to shift,
2692 and AMOUNT the amount to shift by.
2693 Store the result in the rtx TARGET, if that is convenient.
2694 If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2695 Return the rtx for where the value is. */
2698 expand_shift (enum tree_code code
, machine_mode mode
, rtx shifted
,
2699 poly_int64 amount
, rtx target
, int unsignedp
)
2701 return expand_shift_1 (code
, mode
, shifted
,
2702 gen_int_shift_amount (mode
, amount
),
2706 /* Likewise, but return 0 if that cannot be done. */
2709 maybe_expand_shift (enum tree_code code
, machine_mode mode
, rtx shifted
,
2710 int amount
, rtx target
, int unsignedp
)
2712 return expand_shift_1 (code
, mode
,
2713 shifted
, GEN_INT (amount
), target
, unsignedp
, true);
2716 /* Output a shift instruction for expression code CODE,
2717 with SHIFTED being the rtx for the value to shift,
2718 and AMOUNT the tree for the amount to shift by.
2719 Store the result in the rtx TARGET, if that is convenient.
2720 If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2721 Return the rtx for where the value is. */
2724 expand_variable_shift (enum tree_code code
, machine_mode mode
, rtx shifted
,
2725 tree amount
, rtx target
, int unsignedp
)
2727 return expand_shift_1 (code
, mode
,
2728 shifted
, expand_normal (amount
), target
, unsignedp
);
2732 static void synth_mult (struct algorithm
*, unsigned HOST_WIDE_INT
,
2733 const struct mult_cost
*, machine_mode mode
);
2734 static rtx
expand_mult_const (machine_mode
, rtx
, HOST_WIDE_INT
, rtx
,
2735 const struct algorithm
*, enum mult_variant
);
2736 static unsigned HOST_WIDE_INT
invert_mod2n (unsigned HOST_WIDE_INT
, int);
2737 static rtx
extract_high_half (scalar_int_mode
, rtx
);
2738 static rtx
expmed_mult_highpart (scalar_int_mode
, rtx
, rtx
, rtx
, int, int);
2739 static rtx
expmed_mult_highpart_optab (scalar_int_mode
, rtx
, rtx
, rtx
,
2741 /* Compute and return the best algorithm for multiplying by T.
2742 The algorithm must cost less than cost_limit
2743 If retval.cost >= COST_LIMIT, no algorithm was found and all
2744 other field of the returned struct are undefined.
2745 MODE is the machine mode of the multiplication. */
2748 synth_mult (struct algorithm
*alg_out
, unsigned HOST_WIDE_INT t
,
2749 const struct mult_cost
*cost_limit
, machine_mode mode
)
2752 struct algorithm
*alg_in
, *best_alg
;
2753 struct mult_cost best_cost
;
2754 struct mult_cost new_limit
;
2755 int op_cost
, op_latency
;
2756 unsigned HOST_WIDE_INT orig_t
= t
;
2757 unsigned HOST_WIDE_INT q
;
2758 int maxm
, hash_index
;
2759 bool cache_hit
= false;
2760 enum alg_code cache_alg
= alg_zero
;
2761 bool speed
= optimize_insn_for_speed_p ();
2762 scalar_int_mode imode
;
2763 struct alg_hash_entry
*entry_ptr
;
2765 /* Indicate that no algorithm is yet found. If no algorithm
2766 is found, this value will be returned and indicate failure. */
2767 alg_out
->cost
.cost
= cost_limit
->cost
+ 1;
2768 alg_out
->cost
.latency
= cost_limit
->latency
+ 1;
2770 if (cost_limit
->cost
< 0
2771 || (cost_limit
->cost
== 0 && cost_limit
->latency
<= 0))
2774 /* Be prepared for vector modes. */
2775 imode
= as_a
<scalar_int_mode
> (GET_MODE_INNER (mode
));
2777 maxm
= MIN (BITS_PER_WORD
, GET_MODE_BITSIZE (imode
));
2779 /* Restrict the bits of "t" to the multiplication's mode. */
2780 t
&= GET_MODE_MASK (imode
);
2782 /* t == 1 can be done in zero cost. */
2786 alg_out
->cost
.cost
= 0;
2787 alg_out
->cost
.latency
= 0;
2788 alg_out
->op
[0] = alg_m
;
2792 /* t == 0 sometimes has a cost. If it does and it exceeds our limit,
2796 if (MULT_COST_LESS (cost_limit
, zero_cost (speed
)))
2801 alg_out
->cost
.cost
= zero_cost (speed
);
2802 alg_out
->cost
.latency
= zero_cost (speed
);
2803 alg_out
->op
[0] = alg_zero
;
2808 /* We'll be needing a couple extra algorithm structures now. */
2810 alg_in
= XALLOCA (struct algorithm
);
2811 best_alg
= XALLOCA (struct algorithm
);
2812 best_cost
= *cost_limit
;
2814 /* Compute the hash index. */
2815 hash_index
= (t
^ (unsigned int) mode
^ (speed
* 256)) % NUM_ALG_HASH_ENTRIES
;
2817 /* See if we already know what to do for T. */
2818 entry_ptr
= alg_hash_entry_ptr (hash_index
);
2819 if (entry_ptr
->t
== t
2820 && entry_ptr
->mode
== mode
2821 && entry_ptr
->speed
== speed
2822 && entry_ptr
->alg
!= alg_unknown
)
2824 cache_alg
= entry_ptr
->alg
;
2826 if (cache_alg
== alg_impossible
)
2828 /* The cache tells us that it's impossible to synthesize
2829 multiplication by T within entry_ptr->cost. */
2830 if (!CHEAPER_MULT_COST (&entry_ptr
->cost
, cost_limit
))
2831 /* COST_LIMIT is at least as restrictive as the one
2832 recorded in the hash table, in which case we have no
2833 hope of synthesizing a multiplication. Just
2837 /* If we get here, COST_LIMIT is less restrictive than the
2838 one recorded in the hash table, so we may be able to
2839 synthesize a multiplication. Proceed as if we didn't
2840 have the cache entry. */
2844 if (CHEAPER_MULT_COST (cost_limit
, &entry_ptr
->cost
))
2845 /* The cached algorithm shows that this multiplication
2846 requires more cost than COST_LIMIT. Just return. This
2847 way, we don't clobber this cache entry with
2848 alg_impossible but retain useful information. */
2860 goto do_alg_addsub_t_m2
;
2862 case alg_add_factor
:
2863 case alg_sub_factor
:
2864 goto do_alg_addsub_factor
;
2867 goto do_alg_add_t2_m
;
2870 goto do_alg_sub_t2_m
;
2878 /* If we have a group of zero bits at the low-order part of T, try
2879 multiplying by the remaining bits and then doing a shift. */
2884 m
= ctz_or_zero (t
); /* m = number of low zero bits */
2888 /* The function expand_shift will choose between a shift and
2889 a sequence of additions, so the observed cost is given as
2890 MIN (m * add_cost(speed, mode), shift_cost(speed, mode, m)). */
2891 op_cost
= m
* add_cost (speed
, mode
);
2892 if (shift_cost (speed
, mode
, m
) < op_cost
)
2893 op_cost
= shift_cost (speed
, mode
, m
);
2894 new_limit
.cost
= best_cost
.cost
- op_cost
;
2895 new_limit
.latency
= best_cost
.latency
- op_cost
;
2896 synth_mult (alg_in
, q
, &new_limit
, mode
);
2898 alg_in
->cost
.cost
+= op_cost
;
2899 alg_in
->cost
.latency
+= op_cost
;
2900 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2902 best_cost
= alg_in
->cost
;
2903 std::swap (alg_in
, best_alg
);
2904 best_alg
->log
[best_alg
->ops
] = m
;
2905 best_alg
->op
[best_alg
->ops
] = alg_shift
;
2908 /* See if treating ORIG_T as a signed number yields a better
2909 sequence. Try this sequence only for a negative ORIG_T
2910 as it would be useless for a non-negative ORIG_T. */
2911 if ((HOST_WIDE_INT
) orig_t
< 0)
2913 /* Shift ORIG_T as follows because a right shift of a
2914 negative-valued signed type is implementation
2916 q
= ~(~orig_t
>> m
);
2917 /* The function expand_shift will choose between a shift
2918 and a sequence of additions, so the observed cost is
2919 given as MIN (m * add_cost(speed, mode),
2920 shift_cost(speed, mode, m)). */
2921 op_cost
= m
* add_cost (speed
, mode
);
2922 if (shift_cost (speed
, mode
, m
) < op_cost
)
2923 op_cost
= shift_cost (speed
, mode
, m
);
2924 new_limit
.cost
= best_cost
.cost
- op_cost
;
2925 new_limit
.latency
= best_cost
.latency
- op_cost
;
2926 synth_mult (alg_in
, q
, &new_limit
, mode
);
2928 alg_in
->cost
.cost
+= op_cost
;
2929 alg_in
->cost
.latency
+= op_cost
;
2930 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2932 best_cost
= alg_in
->cost
;
2933 std::swap (alg_in
, best_alg
);
2934 best_alg
->log
[best_alg
->ops
] = m
;
2935 best_alg
->op
[best_alg
->ops
] = alg_shift
;
2943 /* If we have an odd number, add or subtract one. */
2946 unsigned HOST_WIDE_INT w
;
2949 for (w
= 1; (w
& t
) != 0; w
<<= 1)
2951 /* If T was -1, then W will be zero after the loop. This is another
2952 case where T ends with ...111. Handling this with (T + 1) and
2953 subtract 1 produces slightly better code and results in algorithm
2954 selection much faster than treating it like the ...0111 case
2958 /* Reject the case where t is 3.
2959 Thus we prefer addition in that case. */
2962 /* T ends with ...111. Multiply by (T + 1) and subtract T. */
2964 op_cost
= add_cost (speed
, mode
);
2965 new_limit
.cost
= best_cost
.cost
- op_cost
;
2966 new_limit
.latency
= best_cost
.latency
- op_cost
;
2967 synth_mult (alg_in
, t
+ 1, &new_limit
, mode
);
2969 alg_in
->cost
.cost
+= op_cost
;
2970 alg_in
->cost
.latency
+= op_cost
;
2971 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2973 best_cost
= alg_in
->cost
;
2974 std::swap (alg_in
, best_alg
);
2975 best_alg
->log
[best_alg
->ops
] = 0;
2976 best_alg
->op
[best_alg
->ops
] = alg_sub_t_m2
;
2981 /* T ends with ...01 or ...011. Multiply by (T - 1) and add T. */
2983 op_cost
= add_cost (speed
, mode
);
2984 new_limit
.cost
= best_cost
.cost
- op_cost
;
2985 new_limit
.latency
= best_cost
.latency
- op_cost
;
2986 synth_mult (alg_in
, t
- 1, &new_limit
, mode
);
2988 alg_in
->cost
.cost
+= op_cost
;
2989 alg_in
->cost
.latency
+= op_cost
;
2990 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2992 best_cost
= alg_in
->cost
;
2993 std::swap (alg_in
, best_alg
);
2994 best_alg
->log
[best_alg
->ops
] = 0;
2995 best_alg
->op
[best_alg
->ops
] = alg_add_t_m2
;
2999 /* We may be able to calculate a * -7, a * -15, a * -31, etc
3000 quickly with a - a * n for some appropriate constant n. */
3001 m
= exact_log2 (-orig_t
+ 1);
3002 if (m
>= 0 && m
< maxm
)
3004 op_cost
= add_cost (speed
, mode
) + shift_cost (speed
, mode
, m
);
3005 /* If the target has a cheap shift-and-subtract insn use
3006 that in preference to a shift insn followed by a sub insn.
3007 Assume that the shift-and-sub is "atomic" with a latency
3008 equal to it's cost, otherwise assume that on superscalar
3009 hardware the shift may be executed concurrently with the
3010 earlier steps in the algorithm. */
3011 if (shiftsub1_cost (speed
, mode
, m
) <= op_cost
)
3013 op_cost
= shiftsub1_cost (speed
, mode
, m
);
3014 op_latency
= op_cost
;
3017 op_latency
= add_cost (speed
, mode
);
3019 new_limit
.cost
= best_cost
.cost
- op_cost
;
3020 new_limit
.latency
= best_cost
.latency
- op_latency
;
3021 synth_mult (alg_in
, (unsigned HOST_WIDE_INT
) (-orig_t
+ 1) >> m
,
3024 alg_in
->cost
.cost
+= op_cost
;
3025 alg_in
->cost
.latency
+= op_latency
;
3026 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
3028 best_cost
= alg_in
->cost
;
3029 std::swap (alg_in
, best_alg
);
3030 best_alg
->log
[best_alg
->ops
] = m
;
3031 best_alg
->op
[best_alg
->ops
] = alg_sub_t_m2
;
3039 /* Look for factors of t of the form
3040 t = q(2**m +- 1), 2 <= m <= floor(log2(t - 1)).
3041 If we find such a factor, we can multiply by t using an algorithm that
3042 multiplies by q, shift the result by m and add/subtract it to itself.
3044 We search for large factors first and loop down, even if large factors
3045 are less probable than small; if we find a large factor we will find a
3046 good sequence quickly, and therefore be able to prune (by decreasing
3047 COST_LIMIT) the search. */
3049 do_alg_addsub_factor
:
3050 for (m
= floor_log2 (t
- 1); m
>= 2; m
--)
3052 unsigned HOST_WIDE_INT d
;
3054 d
= (HOST_WIDE_INT_1U
<< m
) + 1;
3055 if (t
% d
== 0 && t
> d
&& m
< maxm
3056 && (!cache_hit
|| cache_alg
== alg_add_factor
))
3058 op_cost
= add_cost (speed
, mode
) + shift_cost (speed
, mode
, m
);
3059 if (shiftadd_cost (speed
, mode
, m
) <= op_cost
)
3060 op_cost
= shiftadd_cost (speed
, mode
, m
);
3062 op_latency
= op_cost
;
3065 new_limit
.cost
= best_cost
.cost
- op_cost
;
3066 new_limit
.latency
= best_cost
.latency
- op_latency
;
3067 synth_mult (alg_in
, t
/ d
, &new_limit
, mode
);
3069 alg_in
->cost
.cost
+= op_cost
;
3070 alg_in
->cost
.latency
+= op_latency
;
3071 if (alg_in
->cost
.latency
< op_cost
)
3072 alg_in
->cost
.latency
= op_cost
;
3073 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
3075 best_cost
= alg_in
->cost
;
3076 std::swap (alg_in
, best_alg
);
3077 best_alg
->log
[best_alg
->ops
] = m
;
3078 best_alg
->op
[best_alg
->ops
] = alg_add_factor
;
3080 /* Other factors will have been taken care of in the recursion. */
3084 d
= (HOST_WIDE_INT_1U
<< m
) - 1;
3085 if (t
% d
== 0 && t
> d
&& m
< maxm
3086 && (!cache_hit
|| cache_alg
== alg_sub_factor
))
3088 op_cost
= add_cost (speed
, mode
) + shift_cost (speed
, mode
, m
);
3089 if (shiftsub0_cost (speed
, mode
, m
) <= op_cost
)
3090 op_cost
= shiftsub0_cost (speed
, mode
, m
);
3092 op_latency
= op_cost
;
3094 new_limit
.cost
= best_cost
.cost
- op_cost
;
3095 new_limit
.latency
= best_cost
.latency
- op_latency
;
3096 synth_mult (alg_in
, t
/ d
, &new_limit
, mode
);
3098 alg_in
->cost
.cost
+= op_cost
;
3099 alg_in
->cost
.latency
+= op_latency
;
3100 if (alg_in
->cost
.latency
< op_cost
)
3101 alg_in
->cost
.latency
= op_cost
;
3102 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
3104 best_cost
= alg_in
->cost
;
3105 std::swap (alg_in
, best_alg
);
3106 best_alg
->log
[best_alg
->ops
] = m
;
3107 best_alg
->op
[best_alg
->ops
] = alg_sub_factor
;
3115 /* Try shift-and-add (load effective address) instructions,
3116 i.e. do a*3, a*5, a*9. */
3124 op_cost
= shiftadd_cost (speed
, mode
, m
);
3125 new_limit
.cost
= best_cost
.cost
- op_cost
;
3126 new_limit
.latency
= best_cost
.latency
- op_cost
;
3127 synth_mult (alg_in
, (t
- 1) >> m
, &new_limit
, mode
);
3129 alg_in
->cost
.cost
+= op_cost
;
3130 alg_in
->cost
.latency
+= op_cost
;
3131 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
3133 best_cost
= alg_in
->cost
;
3134 std::swap (alg_in
, best_alg
);
3135 best_alg
->log
[best_alg
->ops
] = m
;
3136 best_alg
->op
[best_alg
->ops
] = alg_add_t2_m
;
3147 op_cost
= shiftsub0_cost (speed
, mode
, m
);
3148 new_limit
.cost
= best_cost
.cost
- op_cost
;
3149 new_limit
.latency
= best_cost
.latency
- op_cost
;
3150 synth_mult (alg_in
, (t
+ 1) >> m
, &new_limit
, mode
);
3152 alg_in
->cost
.cost
+= op_cost
;
3153 alg_in
->cost
.latency
+= op_cost
;
3154 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
3156 best_cost
= alg_in
->cost
;
3157 std::swap (alg_in
, best_alg
);
3158 best_alg
->log
[best_alg
->ops
] = m
;
3159 best_alg
->op
[best_alg
->ops
] = alg_sub_t2_m
;
3167 /* If best_cost has not decreased, we have not found any algorithm. */
3168 if (!CHEAPER_MULT_COST (&best_cost
, cost_limit
))
3170 /* We failed to find an algorithm. Record alg_impossible for
3171 this case (that is, <T, MODE, COST_LIMIT>) so that next time
3172 we are asked to find an algorithm for T within the same or
3173 lower COST_LIMIT, we can immediately return to the
3176 entry_ptr
->mode
= mode
;
3177 entry_ptr
->speed
= speed
;
3178 entry_ptr
->alg
= alg_impossible
;
3179 entry_ptr
->cost
= *cost_limit
;
3183 /* Cache the result. */
3187 entry_ptr
->mode
= mode
;
3188 entry_ptr
->speed
= speed
;
3189 entry_ptr
->alg
= best_alg
->op
[best_alg
->ops
];
3190 entry_ptr
->cost
.cost
= best_cost
.cost
;
3191 entry_ptr
->cost
.latency
= best_cost
.latency
;
3194 /* If we are getting a too long sequence for `struct algorithm'
3195 to record, make this search fail. */
3196 if (best_alg
->ops
== MAX_BITS_PER_WORD
)
3199 /* Copy the algorithm from temporary space to the space at alg_out.
3200 We avoid using structure assignment because the majority of
3201 best_alg is normally undefined, and this is a critical function. */
3202 alg_out
->ops
= best_alg
->ops
+ 1;
3203 alg_out
->cost
= best_cost
;
3204 memcpy (alg_out
->op
, best_alg
->op
,
3205 alg_out
->ops
* sizeof *alg_out
->op
);
3206 memcpy (alg_out
->log
, best_alg
->log
,
3207 alg_out
->ops
* sizeof *alg_out
->log
);
3210 /* Find the cheapest way of multiplying a value of mode MODE by VAL.
3211 Try three variations:
3213 - a shift/add sequence based on VAL itself
3214 - a shift/add sequence based on -VAL, followed by a negation
3215 - a shift/add sequence based on VAL - 1, followed by an addition.
3217 Return true if the cheapest of these cost less than MULT_COST,
3218 describing the algorithm in *ALG and final fixup in *VARIANT. */
3221 choose_mult_variant (machine_mode mode
, HOST_WIDE_INT val
,
3222 struct algorithm
*alg
, enum mult_variant
*variant
,
3225 struct algorithm alg2
;
3226 struct mult_cost limit
;
3228 bool speed
= optimize_insn_for_speed_p ();
3230 /* Fail quickly for impossible bounds. */
3234 /* Ensure that mult_cost provides a reasonable upper bound.
3235 Any constant multiplication can be performed with less
3236 than 2 * bits additions. */
3237 op_cost
= 2 * GET_MODE_UNIT_BITSIZE (mode
) * add_cost (speed
, mode
);
3238 if (mult_cost
> op_cost
)
3239 mult_cost
= op_cost
;
3241 *variant
= basic_variant
;
3242 limit
.cost
= mult_cost
;
3243 limit
.latency
= mult_cost
;
3244 synth_mult (alg
, val
, &limit
, mode
);
3246 /* This works only if the inverted value actually fits in an
3248 if (HOST_BITS_PER_INT
>= GET_MODE_UNIT_BITSIZE (mode
))
3250 op_cost
= neg_cost (speed
, mode
);
3251 if (MULT_COST_LESS (&alg
->cost
, mult_cost
))
3253 limit
.cost
= alg
->cost
.cost
- op_cost
;
3254 limit
.latency
= alg
->cost
.latency
- op_cost
;
3258 limit
.cost
= mult_cost
- op_cost
;
3259 limit
.latency
= mult_cost
- op_cost
;
3262 synth_mult (&alg2
, -val
, &limit
, mode
);
3263 alg2
.cost
.cost
+= op_cost
;
3264 alg2
.cost
.latency
+= op_cost
;
3265 if (CHEAPER_MULT_COST (&alg2
.cost
, &alg
->cost
))
3266 *alg
= alg2
, *variant
= negate_variant
;
3269 /* This proves very useful for division-by-constant. */
3270 op_cost
= add_cost (speed
, mode
);
3271 if (MULT_COST_LESS (&alg
->cost
, mult_cost
))
3273 limit
.cost
= alg
->cost
.cost
- op_cost
;
3274 limit
.latency
= alg
->cost
.latency
- op_cost
;
3278 limit
.cost
= mult_cost
- op_cost
;
3279 limit
.latency
= mult_cost
- op_cost
;
3282 synth_mult (&alg2
, val
- 1, &limit
, mode
);
3283 alg2
.cost
.cost
+= op_cost
;
3284 alg2
.cost
.latency
+= op_cost
;
3285 if (CHEAPER_MULT_COST (&alg2
.cost
, &alg
->cost
))
3286 *alg
= alg2
, *variant
= add_variant
;
3288 return MULT_COST_LESS (&alg
->cost
, mult_cost
);
3291 /* A subroutine of expand_mult, used for constant multiplications.
3292 Multiply OP0 by VAL in mode MODE, storing the result in TARGET if
3293 convenient. Use the shift/add sequence described by ALG and apply
3294 the final fixup specified by VARIANT. */
3297 expand_mult_const (machine_mode mode
, rtx op0
, HOST_WIDE_INT val
,
3298 rtx target
, const struct algorithm
*alg
,
3299 enum mult_variant variant
)
3301 unsigned HOST_WIDE_INT val_so_far
;
3307 /* Avoid referencing memory over and over and invalid sharing
3309 op0
= force_reg (mode
, op0
);
3311 /* ACCUM starts out either as OP0 or as a zero, depending on
3312 the first operation. */
3314 if (alg
->op
[0] == alg_zero
)
3316 accum
= copy_to_mode_reg (mode
, CONST0_RTX (mode
));
3319 else if (alg
->op
[0] == alg_m
)
3321 accum
= copy_to_mode_reg (mode
, op0
);
3327 for (opno
= 1; opno
< alg
->ops
; opno
++)
3329 int log
= alg
->log
[opno
];
3330 rtx shift_subtarget
= optimize
? 0 : accum
;
3332 = (opno
== alg
->ops
- 1 && target
!= 0 && variant
!= add_variant
3335 rtx accum_target
= optimize
? 0 : accum
;
3338 switch (alg
->op
[opno
])
3341 tem
= expand_shift (LSHIFT_EXPR
, mode
, accum
, log
, NULL_RTX
, 0);
3342 /* REG_EQUAL note will be attached to the following insn. */
3343 emit_move_insn (accum
, tem
);
3348 tem
= expand_shift (LSHIFT_EXPR
, mode
, op0
, log
, NULL_RTX
, 0);
3349 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, tem
),
3350 add_target
? add_target
: accum_target
);
3351 val_so_far
+= HOST_WIDE_INT_1U
<< log
;
3355 tem
= expand_shift (LSHIFT_EXPR
, mode
, op0
, log
, NULL_RTX
, 0);
3356 accum
= force_operand (gen_rtx_MINUS (mode
, accum
, tem
),
3357 add_target
? add_target
: accum_target
);
3358 val_so_far
-= HOST_WIDE_INT_1U
<< log
;
3362 accum
= expand_shift (LSHIFT_EXPR
, mode
, accum
,
3363 log
, shift_subtarget
, 0);
3364 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, op0
),
3365 add_target
? add_target
: accum_target
);
3366 val_so_far
= (val_so_far
<< log
) + 1;
3370 accum
= expand_shift (LSHIFT_EXPR
, mode
, accum
,
3371 log
, shift_subtarget
, 0);
3372 accum
= force_operand (gen_rtx_MINUS (mode
, accum
, op0
),
3373 add_target
? add_target
: accum_target
);
3374 val_so_far
= (val_so_far
<< log
) - 1;
3377 case alg_add_factor
:
3378 tem
= expand_shift (LSHIFT_EXPR
, mode
, accum
, log
, NULL_RTX
, 0);
3379 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, tem
),
3380 add_target
? add_target
: accum_target
);
3381 val_so_far
+= val_so_far
<< log
;
3384 case alg_sub_factor
:
3385 tem
= expand_shift (LSHIFT_EXPR
, mode
, accum
, log
, NULL_RTX
, 0);
3386 accum
= force_operand (gen_rtx_MINUS (mode
, tem
, accum
),
3388 ? add_target
: (optimize
? 0 : tem
)));
3389 val_so_far
= (val_so_far
<< log
) - val_so_far
;
3396 if (SCALAR_INT_MODE_P (mode
))
3398 /* Write a REG_EQUAL note on the last insn so that we can cse
3399 multiplication sequences. Note that if ACCUM is a SUBREG,
3400 we've set the inner register and must properly indicate that. */
3401 tem
= op0
, nmode
= mode
;
3402 accum_inner
= accum
;
3403 if (GET_CODE (accum
) == SUBREG
)
3405 accum_inner
= SUBREG_REG (accum
);
3406 nmode
= GET_MODE (accum_inner
);
3407 tem
= gen_lowpart (nmode
, op0
);
3410 /* Don't add a REG_EQUAL note if tem is a paradoxical SUBREG.
3411 In that case, only the low bits of accum would be guaranteed to
3412 be equal to the content of the REG_EQUAL note, the upper bits
3414 if (!paradoxical_subreg_p (tem
))
3416 insn
= get_last_insn ();
3417 wide_int wval_so_far
3418 = wi::uhwi (val_so_far
,
3419 GET_MODE_PRECISION (as_a
<scalar_mode
> (nmode
)));
3420 rtx c
= immed_wide_int_const (wval_so_far
, nmode
);
3421 set_dst_reg_note (insn
, REG_EQUAL
, gen_rtx_MULT (nmode
, tem
, c
),
3427 if (variant
== negate_variant
)
3429 val_so_far
= -val_so_far
;
3430 accum
= expand_unop (mode
, neg_optab
, accum
, target
, 0);
3432 else if (variant
== add_variant
)
3434 val_so_far
= val_so_far
+ 1;
3435 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, op0
), target
);
3438 /* Compare only the bits of val and val_so_far that are significant
3439 in the result mode, to avoid sign-/zero-extension confusion. */
3440 nmode
= GET_MODE_INNER (mode
);
3441 val
&= GET_MODE_MASK (nmode
);
3442 val_so_far
&= GET_MODE_MASK (nmode
);
3443 gcc_assert (val
== (HOST_WIDE_INT
) val_so_far
);
3448 /* Perform a multiplication and return an rtx for the result.
3449 MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3450 TARGET is a suggestion for where to store the result (an rtx).
3452 We check specially for a constant integer as OP1.
3453 If you want this check for OP0 as well, then before calling
3454 you should swap the two operands if OP0 would be constant. */
3457 expand_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3458 int unsignedp
, bool no_libcall
)
3460 enum mult_variant variant
;
3461 struct algorithm algorithm
;
3464 bool speed
= optimize_insn_for_speed_p ();
3465 bool do_trapv
= flag_trapv
&& SCALAR_INT_MODE_P (mode
) && !unsignedp
;
3467 if (CONSTANT_P (op0
))
3468 std::swap (op0
, op1
);
3470 /* For vectors, there are several simplifications that can be made if
3471 all elements of the vector constant are identical. */
3472 scalar_op1
= unwrap_const_vec_duplicate (op1
);
3474 if (INTEGRAL_MODE_P (mode
))
3477 HOST_WIDE_INT coeff
;
3481 if (op1
== CONST0_RTX (mode
))
3483 if (op1
== CONST1_RTX (mode
))
3485 if (op1
== CONSTM1_RTX (mode
))
3486 return expand_unop (mode
, do_trapv
? negv_optab
: neg_optab
,
3492 /* If mode is integer vector mode, check if the backend supports
3493 vector lshift (by scalar or vector) at all. If not, we can't use
3494 synthetized multiply. */
3495 if (GET_MODE_CLASS (mode
) == MODE_VECTOR_INT
3496 && optab_handler (vashl_optab
, mode
) == CODE_FOR_nothing
3497 && optab_handler (ashl_optab
, mode
) == CODE_FOR_nothing
)
3500 /* These are the operations that are potentially turned into
3501 a sequence of shifts and additions. */
3502 mode_bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
3504 /* synth_mult does an `unsigned int' multiply. As long as the mode is
3505 less than or equal in size to `unsigned int' this doesn't matter.
3506 If the mode is larger than `unsigned int', then synth_mult works
3507 only if the constant value exactly fits in an `unsigned int' without
3508 any truncation. This means that multiplying by negative values does
3509 not work; results are off by 2^32 on a 32 bit machine. */
3510 if (CONST_INT_P (scalar_op1
))
3512 coeff
= INTVAL (scalar_op1
);
3515 #if TARGET_SUPPORTS_WIDE_INT
3516 else if (CONST_WIDE_INT_P (scalar_op1
))
3518 else if (CONST_DOUBLE_AS_INT_P (scalar_op1
))
3521 int shift
= wi::exact_log2 (rtx_mode_t (scalar_op1
, mode
));
3522 /* Perfect power of 2 (other than 1, which is handled above). */
3524 return expand_shift (LSHIFT_EXPR
, mode
, op0
,
3525 shift
, target
, unsignedp
);
3532 /* We used to test optimize here, on the grounds that it's better to
3533 produce a smaller program when -O is not used. But this causes
3534 such a terrible slowdown sometimes that it seems better to always
3537 /* Special case powers of two. */
3538 if (EXACT_POWER_OF_2_OR_ZERO_P (coeff
)
3539 && !(is_neg
&& mode_bitsize
> HOST_BITS_PER_WIDE_INT
))
3540 return expand_shift (LSHIFT_EXPR
, mode
, op0
,
3541 floor_log2 (coeff
), target
, unsignedp
);
3543 fake_reg
= gen_raw_REG (mode
, LAST_VIRTUAL_REGISTER
+ 1);
3545 /* Attempt to handle multiplication of DImode values by negative
3546 coefficients, by performing the multiplication by a positive
3547 multiplier and then inverting the result. */
3548 if (is_neg
&& mode_bitsize
> HOST_BITS_PER_WIDE_INT
)
3550 /* Its safe to use -coeff even for INT_MIN, as the
3551 result is interpreted as an unsigned coefficient.
3552 Exclude cost of op0 from max_cost to match the cost
3553 calculation of the synth_mult. */
3554 coeff
= -(unsigned HOST_WIDE_INT
) coeff
;
3555 max_cost
= (set_src_cost (gen_rtx_MULT (mode
, fake_reg
, op1
),
3557 - neg_cost (speed
, mode
));
3561 /* Special case powers of two. */
3562 if (EXACT_POWER_OF_2_OR_ZERO_P (coeff
))
3564 rtx temp
= expand_shift (LSHIFT_EXPR
, mode
, op0
,
3565 floor_log2 (coeff
), target
, unsignedp
);
3566 return expand_unop (mode
, neg_optab
, temp
, target
, 0);
3569 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
,
3572 rtx temp
= expand_mult_const (mode
, op0
, coeff
, NULL_RTX
,
3573 &algorithm
, variant
);
3574 return expand_unop (mode
, neg_optab
, temp
, target
, 0);
3579 /* Exclude cost of op0 from max_cost to match the cost
3580 calculation of the synth_mult. */
3581 max_cost
= set_src_cost (gen_rtx_MULT (mode
, fake_reg
, op1
), mode
, speed
);
3582 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
, max_cost
))
3583 return expand_mult_const (mode
, op0
, coeff
, target
,
3584 &algorithm
, variant
);
3588 /* Expand x*2.0 as x+x. */
3589 if (CONST_DOUBLE_AS_FLOAT_P (scalar_op1
)
3590 && real_equal (CONST_DOUBLE_REAL_VALUE (scalar_op1
), &dconst2
))
3592 op0
= force_reg (GET_MODE (op0
), op0
);
3593 return expand_binop (mode
, add_optab
, op0
, op0
,
3595 no_libcall
? OPTAB_WIDEN
: OPTAB_LIB_WIDEN
);
3598 /* This used to use umul_optab if unsigned, but for non-widening multiply
3599 there is no difference between signed and unsigned. */
3600 op0
= expand_binop (mode
, do_trapv
? smulv_optab
: smul_optab
,
3601 op0
, op1
, target
, unsignedp
,
3602 no_libcall
? OPTAB_WIDEN
: OPTAB_LIB_WIDEN
);
3603 gcc_assert (op0
|| no_libcall
);
3607 /* Return a cost estimate for multiplying a register by the given
3608 COEFFicient in the given MODE and SPEED. */
3611 mult_by_coeff_cost (HOST_WIDE_INT coeff
, machine_mode mode
, bool speed
)
3614 struct algorithm algorithm
;
3615 enum mult_variant variant
;
3617 rtx fake_reg
= gen_raw_REG (mode
, LAST_VIRTUAL_REGISTER
+ 1);
3618 max_cost
= set_src_cost (gen_rtx_MULT (mode
, fake_reg
, fake_reg
),
3620 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
, max_cost
))
3621 return algorithm
.cost
.cost
;
3626 /* Perform a widening multiplication and return an rtx for the result.
3627 MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3628 TARGET is a suggestion for where to store the result (an rtx).
3629 THIS_OPTAB is the optab we should use, it must be either umul_widen_optab
3630 or smul_widen_optab.
3632 We check specially for a constant integer as OP1, comparing the
3633 cost of a widening multiply against the cost of a sequence of shifts
3637 expand_widening_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3638 int unsignedp
, optab this_optab
)
3640 bool speed
= optimize_insn_for_speed_p ();
3643 if (CONST_INT_P (op1
)
3644 && GET_MODE (op0
) != VOIDmode
3645 && (cop1
= convert_modes (mode
, GET_MODE (op0
), op1
,
3646 this_optab
== umul_widen_optab
))
3647 && CONST_INT_P (cop1
)
3648 && (INTVAL (cop1
) >= 0
3649 || HWI_COMPUTABLE_MODE_P (mode
)))
3651 HOST_WIDE_INT coeff
= INTVAL (cop1
);
3653 enum mult_variant variant
;
3654 struct algorithm algorithm
;
3657 return CONST0_RTX (mode
);
3659 /* Special case powers of two. */
3660 if (EXACT_POWER_OF_2_OR_ZERO_P (coeff
))
3662 op0
= convert_to_mode (mode
, op0
, this_optab
== umul_widen_optab
);
3663 return expand_shift (LSHIFT_EXPR
, mode
, op0
,
3664 floor_log2 (coeff
), target
, unsignedp
);
3667 /* Exclude cost of op0 from max_cost to match the cost
3668 calculation of the synth_mult. */
3669 max_cost
= mul_widen_cost (speed
, mode
);
3670 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
,
3673 op0
= convert_to_mode (mode
, op0
, this_optab
== umul_widen_optab
);
3674 return expand_mult_const (mode
, op0
, coeff
, target
,
3675 &algorithm
, variant
);
3678 return expand_binop (mode
, this_optab
, op0
, op1
, target
,
3679 unsignedp
, OPTAB_LIB_WIDEN
);
3682 /* Choose a minimal N + 1 bit approximation to 1/D that can be used to
3683 replace division by D, and put the least significant N bits of the result
3684 in *MULTIPLIER_PTR and return the most significant bit.
3686 The width of operations is N (should be <= HOST_BITS_PER_WIDE_INT), the
3687 needed precision is in PRECISION (should be <= N).
3689 PRECISION should be as small as possible so this function can choose
3690 multiplier more freely.
3692 The rounded-up logarithm of D is placed in *lgup_ptr. A shift count that
3693 is to be used for a final right shift is placed in *POST_SHIFT_PTR.
3695 Using this function, x/D will be equal to (x * m) >> (*POST_SHIFT_PTR),
3696 where m is the full HOST_BITS_PER_WIDE_INT + 1 bit multiplier. */
3698 unsigned HOST_WIDE_INT
3699 choose_multiplier (unsigned HOST_WIDE_INT d
, int n
, int precision
,
3700 unsigned HOST_WIDE_INT
*multiplier_ptr
,
3701 int *post_shift_ptr
, int *lgup_ptr
)
3703 int lgup
, post_shift
;
3706 /* lgup = ceil(log2(divisor)); */
3707 lgup
= ceil_log2 (d
);
3709 gcc_assert (lgup
<= n
);
3712 pow2
= n
+ lgup
- precision
;
3714 /* mlow = 2^(N + lgup)/d */
3715 wide_int val
= wi::set_bit_in_zero (pow
, HOST_BITS_PER_DOUBLE_INT
);
3716 wide_int mlow
= wi::udiv_trunc (val
, d
);
3718 /* mhigh = (2^(N + lgup) + 2^(N + lgup - precision))/d */
3719 val
|= wi::set_bit_in_zero (pow2
, HOST_BITS_PER_DOUBLE_INT
);
3720 wide_int mhigh
= wi::udiv_trunc (val
, d
);
3722 /* If precision == N, then mlow, mhigh exceed 2^N
3723 (but they do not exceed 2^(N+1)). */
3725 /* Reduce to lowest terms. */
3726 for (post_shift
= lgup
; post_shift
> 0; post_shift
--)
3728 unsigned HOST_WIDE_INT ml_lo
= wi::extract_uhwi (mlow
, 1,
3729 HOST_BITS_PER_WIDE_INT
);
3730 unsigned HOST_WIDE_INT mh_lo
= wi::extract_uhwi (mhigh
, 1,
3731 HOST_BITS_PER_WIDE_INT
);
3735 mlow
= wi::uhwi (ml_lo
, HOST_BITS_PER_DOUBLE_INT
);
3736 mhigh
= wi::uhwi (mh_lo
, HOST_BITS_PER_DOUBLE_INT
);
3739 *post_shift_ptr
= post_shift
;
3741 if (n
< HOST_BITS_PER_WIDE_INT
)
3743 unsigned HOST_WIDE_INT mask
= (HOST_WIDE_INT_1U
<< n
) - 1;
3744 *multiplier_ptr
= mhigh
.to_uhwi () & mask
;
3745 return mhigh
.to_uhwi () > mask
;
3749 *multiplier_ptr
= mhigh
.to_uhwi ();
3750 return wi::extract_uhwi (mhigh
, HOST_BITS_PER_WIDE_INT
, 1);
3754 /* Compute the inverse of X mod 2**n, i.e., find Y such that X * Y is
3755 congruent to 1 (mod 2**N). */
3757 static unsigned HOST_WIDE_INT
3758 invert_mod2n (unsigned HOST_WIDE_INT x
, int n
)
3760 /* Solve x*y == 1 (mod 2^n), where x is odd. Return y. */
3762 /* The algorithm notes that the choice y = x satisfies
3763 x*y == 1 mod 2^3, since x is assumed odd.
3764 Each iteration doubles the number of bits of significance in y. */
3766 unsigned HOST_WIDE_INT mask
;
3767 unsigned HOST_WIDE_INT y
= x
;
3770 mask
= (n
== HOST_BITS_PER_WIDE_INT
3772 : (HOST_WIDE_INT_1U
<< n
) - 1);
3776 y
= y
* (2 - x
*y
) & mask
; /* Modulo 2^N */
3782 /* Emit code to adjust ADJ_OPERAND after multiplication of wrong signedness
3783 flavor of OP0 and OP1. ADJ_OPERAND is already the high half of the
3784 product OP0 x OP1. If UNSIGNEDP is nonzero, adjust the signed product
3785 to become unsigned, if UNSIGNEDP is zero, adjust the unsigned product to
3788 The result is put in TARGET if that is convenient.
3790 MODE is the mode of operation. */
3793 expand_mult_highpart_adjust (scalar_int_mode mode
, rtx adj_operand
, rtx op0
,
3794 rtx op1
, rtx target
, int unsignedp
)
3797 enum rtx_code adj_code
= unsignedp
? PLUS
: MINUS
;
3799 tem
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3800 GET_MODE_BITSIZE (mode
) - 1, NULL_RTX
, 0);
3801 tem
= expand_and (mode
, tem
, op1
, NULL_RTX
);
3803 = force_operand (gen_rtx_fmt_ee (adj_code
, mode
, adj_operand
, tem
),
3806 tem
= expand_shift (RSHIFT_EXPR
, mode
, op1
,
3807 GET_MODE_BITSIZE (mode
) - 1, NULL_RTX
, 0);
3808 tem
= expand_and (mode
, tem
, op0
, NULL_RTX
);
3809 target
= force_operand (gen_rtx_fmt_ee (adj_code
, mode
, adj_operand
, tem
),
3815 /* Subroutine of expmed_mult_highpart. Return the MODE high part of OP. */
3818 extract_high_half (scalar_int_mode mode
, rtx op
)
3820 if (mode
== word_mode
)
3821 return gen_highpart (mode
, op
);
3823 scalar_int_mode wider_mode
= GET_MODE_WIDER_MODE (mode
).require ();
3825 op
= expand_shift (RSHIFT_EXPR
, wider_mode
, op
,
3826 GET_MODE_BITSIZE (mode
), 0, 1);
3827 return convert_modes (mode
, wider_mode
, op
, 0);
3830 /* Like expmed_mult_highpart, but only consider using a multiplication
3831 optab. OP1 is an rtx for the constant operand. */
3834 expmed_mult_highpart_optab (scalar_int_mode mode
, rtx op0
, rtx op1
,
3835 rtx target
, int unsignedp
, int max_cost
)
3837 rtx narrow_op1
= gen_int_mode (INTVAL (op1
), mode
);
3841 bool speed
= optimize_insn_for_speed_p ();
3843 scalar_int_mode wider_mode
= GET_MODE_WIDER_MODE (mode
).require ();
3845 size
= GET_MODE_BITSIZE (mode
);
3847 /* Firstly, try using a multiplication insn that only generates the needed
3848 high part of the product, and in the sign flavor of unsignedp. */
3849 if (mul_highpart_cost (speed
, mode
) < max_cost
)
3851 moptab
= unsignedp
? umul_highpart_optab
: smul_highpart_optab
;
3852 tem
= expand_binop (mode
, moptab
, op0
, narrow_op1
, target
,
3853 unsignedp
, OPTAB_DIRECT
);
3858 /* Secondly, same as above, but use sign flavor opposite of unsignedp.
3859 Need to adjust the result after the multiplication. */
3860 if (size
- 1 < BITS_PER_WORD
3861 && (mul_highpart_cost (speed
, mode
)
3862 + 2 * shift_cost (speed
, mode
, size
-1)
3863 + 4 * add_cost (speed
, mode
) < max_cost
))
3865 moptab
= unsignedp
? smul_highpart_optab
: umul_highpart_optab
;
3866 tem
= expand_binop (mode
, moptab
, op0
, narrow_op1
, target
,
3867 unsignedp
, OPTAB_DIRECT
);
3869 /* We used the wrong signedness. Adjust the result. */
3870 return expand_mult_highpart_adjust (mode
, tem
, op0
, narrow_op1
,
3874 /* Try widening multiplication. */
3875 moptab
= unsignedp
? umul_widen_optab
: smul_widen_optab
;
3876 if (convert_optab_handler (moptab
, wider_mode
, mode
) != CODE_FOR_nothing
3877 && mul_widen_cost (speed
, wider_mode
) < max_cost
)
3879 tem
= expand_binop (wider_mode
, moptab
, op0
, narrow_op1
, 0,
3880 unsignedp
, OPTAB_WIDEN
);
3882 return extract_high_half (mode
, tem
);
3885 /* Try widening the mode and perform a non-widening multiplication. */
3886 if (optab_handler (smul_optab
, wider_mode
) != CODE_FOR_nothing
3887 && size
- 1 < BITS_PER_WORD
3888 && (mul_cost (speed
, wider_mode
) + shift_cost (speed
, mode
, size
-1)
3894 /* We need to widen the operands, for example to ensure the
3895 constant multiplier is correctly sign or zero extended.
3896 Use a sequence to clean-up any instructions emitted by
3897 the conversions if things don't work out. */
3899 wop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
3900 wop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
3901 tem
= expand_binop (wider_mode
, smul_optab
, wop0
, wop1
, 0,
3902 unsignedp
, OPTAB_WIDEN
);
3903 insns
= get_insns ();
3909 return extract_high_half (mode
, tem
);
3913 /* Try widening multiplication of opposite signedness, and adjust. */
3914 moptab
= unsignedp
? smul_widen_optab
: umul_widen_optab
;
3915 if (convert_optab_handler (moptab
, wider_mode
, mode
) != CODE_FOR_nothing
3916 && size
- 1 < BITS_PER_WORD
3917 && (mul_widen_cost (speed
, wider_mode
)
3918 + 2 * shift_cost (speed
, mode
, size
-1)
3919 + 4 * add_cost (speed
, mode
) < max_cost
))
3921 tem
= expand_binop (wider_mode
, moptab
, op0
, narrow_op1
,
3922 NULL_RTX
, ! unsignedp
, OPTAB_WIDEN
);
3925 tem
= extract_high_half (mode
, tem
);
3926 /* We used the wrong signedness. Adjust the result. */
3927 return expand_mult_highpart_adjust (mode
, tem
, op0
, narrow_op1
,
3935 /* Emit code to multiply OP0 and OP1 (where OP1 is an integer constant),
3936 putting the high half of the result in TARGET if that is convenient,
3937 and return where the result is. If the operation cannot be performed,
3940 MODE is the mode of operation and result.
3942 UNSIGNEDP nonzero means unsigned multiply.
3944 MAX_COST is the total allowed cost for the expanded RTL. */
3947 expmed_mult_highpart (scalar_int_mode mode
, rtx op0
, rtx op1
,
3948 rtx target
, int unsignedp
, int max_cost
)
3950 unsigned HOST_WIDE_INT cnst1
;
3952 bool sign_adjust
= false;
3953 enum mult_variant variant
;
3954 struct algorithm alg
;
3956 bool speed
= optimize_insn_for_speed_p ();
3958 /* We can't support modes wider than HOST_BITS_PER_INT. */
3959 gcc_assert (HWI_COMPUTABLE_MODE_P (mode
));
3961 cnst1
= INTVAL (op1
) & GET_MODE_MASK (mode
);
3963 /* We can't optimize modes wider than BITS_PER_WORD.
3964 ??? We might be able to perform double-word arithmetic if
3965 mode == word_mode, however all the cost calculations in
3966 synth_mult etc. assume single-word operations. */
3967 scalar_int_mode wider_mode
= GET_MODE_WIDER_MODE (mode
).require ();
3968 if (GET_MODE_BITSIZE (wider_mode
) > BITS_PER_WORD
)
3969 return expmed_mult_highpart_optab (mode
, op0
, op1
, target
,
3970 unsignedp
, max_cost
);
3972 extra_cost
= shift_cost (speed
, mode
, GET_MODE_BITSIZE (mode
) - 1);
3974 /* Check whether we try to multiply by a negative constant. */
3975 if (!unsignedp
&& ((cnst1
>> (GET_MODE_BITSIZE (mode
) - 1)) & 1))
3978 extra_cost
+= add_cost (speed
, mode
);
3981 /* See whether shift/add multiplication is cheap enough. */
3982 if (choose_mult_variant (wider_mode
, cnst1
, &alg
, &variant
,
3983 max_cost
- extra_cost
))
3985 /* See whether the specialized multiplication optabs are
3986 cheaper than the shift/add version. */
3987 tem
= expmed_mult_highpart_optab (mode
, op0
, op1
, target
, unsignedp
,
3988 alg
.cost
.cost
+ extra_cost
);
3992 tem
= convert_to_mode (wider_mode
, op0
, unsignedp
);
3993 tem
= expand_mult_const (wider_mode
, tem
, cnst1
, 0, &alg
, variant
);
3994 tem
= extract_high_half (mode
, tem
);
3996 /* Adjust result for signedness. */
3998 tem
= force_operand (gen_rtx_MINUS (mode
, tem
, op0
), tem
);
4002 return expmed_mult_highpart_optab (mode
, op0
, op1
, target
,
4003 unsignedp
, max_cost
);
4007 /* Expand signed modulus of OP0 by a power of two D in mode MODE. */
4010 expand_smod_pow2 (scalar_int_mode mode
, rtx op0
, HOST_WIDE_INT d
)
4012 rtx result
, temp
, shift
;
4013 rtx_code_label
*label
;
4015 int prec
= GET_MODE_PRECISION (mode
);
4017 logd
= floor_log2 (d
);
4018 result
= gen_reg_rtx (mode
);
4020 /* Avoid conditional branches when they're expensive. */
4021 if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2
4022 && optimize_insn_for_speed_p ())
4024 rtx signmask
= emit_store_flag (result
, LT
, op0
, const0_rtx
,
4028 HOST_WIDE_INT masklow
= (HOST_WIDE_INT_1
<< logd
) - 1;
4029 signmask
= force_reg (mode
, signmask
);
4030 shift
= gen_int_shift_amount (mode
, GET_MODE_BITSIZE (mode
) - logd
);
4032 /* Use the rtx_cost of a LSHIFTRT instruction to determine
4033 which instruction sequence to use. If logical right shifts
4034 are expensive the use 2 XORs, 2 SUBs and an AND, otherwise
4035 use a LSHIFTRT, 1 ADD, 1 SUB and an AND. */
4037 temp
= gen_rtx_LSHIFTRT (mode
, result
, shift
);
4038 if (optab_handler (lshr_optab
, mode
) == CODE_FOR_nothing
4039 || (set_src_cost (temp
, mode
, optimize_insn_for_speed_p ())
4040 > COSTS_N_INSNS (2)))
4042 temp
= expand_binop (mode
, xor_optab
, op0
, signmask
,
4043 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4044 temp
= expand_binop (mode
, sub_optab
, temp
, signmask
,
4045 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4046 temp
= expand_binop (mode
, and_optab
, temp
,
4047 gen_int_mode (masklow
, mode
),
4048 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4049 temp
= expand_binop (mode
, xor_optab
, temp
, signmask
,
4050 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4051 temp
= expand_binop (mode
, sub_optab
, temp
, signmask
,
4052 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4056 signmask
= expand_binop (mode
, lshr_optab
, signmask
, shift
,
4057 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4058 signmask
= force_reg (mode
, signmask
);
4060 temp
= expand_binop (mode
, add_optab
, op0
, signmask
,
4061 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4062 temp
= expand_binop (mode
, and_optab
, temp
,
4063 gen_int_mode (masklow
, mode
),
4064 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4065 temp
= expand_binop (mode
, sub_optab
, temp
, signmask
,
4066 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4072 /* Mask contains the mode's signbit and the significant bits of the
4073 modulus. By including the signbit in the operation, many targets
4074 can avoid an explicit compare operation in the following comparison
4076 wide_int mask
= wi::mask (logd
, false, prec
);
4077 mask
= wi::set_bit (mask
, prec
- 1);
4079 temp
= expand_binop (mode
, and_optab
, op0
,
4080 immed_wide_int_const (mask
, mode
),
4081 result
, 1, OPTAB_LIB_WIDEN
);
4083 emit_move_insn (result
, temp
);
4085 label
= gen_label_rtx ();
4086 do_cmp_and_jump (result
, const0_rtx
, GE
, mode
, label
);
4088 temp
= expand_binop (mode
, sub_optab
, result
, const1_rtx
, result
,
4089 0, OPTAB_LIB_WIDEN
);
4091 mask
= wi::mask (logd
, true, prec
);
4092 temp
= expand_binop (mode
, ior_optab
, temp
,
4093 immed_wide_int_const (mask
, mode
),
4094 result
, 1, OPTAB_LIB_WIDEN
);
4095 temp
= expand_binop (mode
, add_optab
, temp
, const1_rtx
, result
,
4096 0, OPTAB_LIB_WIDEN
);
4098 emit_move_insn (result
, temp
);
4103 /* Expand signed division of OP0 by a power of two D in mode MODE.
4104 This routine is only called for positive values of D. */
4107 expand_sdiv_pow2 (scalar_int_mode mode
, rtx op0
, HOST_WIDE_INT d
)
4110 rtx_code_label
*label
;
4113 logd
= floor_log2 (d
);
4116 && BRANCH_COST (optimize_insn_for_speed_p (),
4119 temp
= gen_reg_rtx (mode
);
4120 temp
= emit_store_flag (temp
, LT
, op0
, const0_rtx
, mode
, 0, 1);
4121 if (temp
!= NULL_RTX
)
4123 temp
= expand_binop (mode
, add_optab
, temp
, op0
, NULL_RTX
,
4124 0, OPTAB_LIB_WIDEN
);
4125 return expand_shift (RSHIFT_EXPR
, mode
, temp
, logd
, NULL_RTX
, 0);
4129 if (HAVE_conditional_move
4130 && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2)
4135 temp2
= copy_to_mode_reg (mode
, op0
);
4136 temp
= expand_binop (mode
, add_optab
, temp2
, gen_int_mode (d
- 1, mode
),
4137 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
4138 temp
= force_reg (mode
, temp
);
4140 /* Construct "temp2 = (temp2 < 0) ? temp : temp2". */
4141 temp2
= emit_conditional_move (temp2
, { LT
, temp2
, const0_rtx
, mode
},
4142 temp
, temp2
, mode
, 0);
4145 rtx_insn
*seq
= get_insns ();
4148 return expand_shift (RSHIFT_EXPR
, mode
, temp2
, logd
, NULL_RTX
, 0);
4153 if (BRANCH_COST (optimize_insn_for_speed_p (),
4156 int ushift
= GET_MODE_BITSIZE (mode
) - logd
;
4158 temp
= gen_reg_rtx (mode
);
4159 temp
= emit_store_flag (temp
, LT
, op0
, const0_rtx
, mode
, 0, -1);
4160 if (temp
!= NULL_RTX
)
4162 if (GET_MODE_BITSIZE (mode
) >= BITS_PER_WORD
4163 || shift_cost (optimize_insn_for_speed_p (), mode
, ushift
)
4164 > COSTS_N_INSNS (1))
4165 temp
= expand_binop (mode
, and_optab
, temp
,
4166 gen_int_mode (d
- 1, mode
),
4167 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
4169 temp
= expand_shift (RSHIFT_EXPR
, mode
, temp
,
4170 ushift
, NULL_RTX
, 1);
4171 temp
= expand_binop (mode
, add_optab
, temp
, op0
, NULL_RTX
,
4172 0, OPTAB_LIB_WIDEN
);
4173 return expand_shift (RSHIFT_EXPR
, mode
, temp
, logd
, NULL_RTX
, 0);
4177 label
= gen_label_rtx ();
4178 temp
= copy_to_mode_reg (mode
, op0
);
4179 do_cmp_and_jump (temp
, const0_rtx
, GE
, mode
, label
);
4180 expand_inc (temp
, gen_int_mode (d
- 1, mode
));
4182 return expand_shift (RSHIFT_EXPR
, mode
, temp
, logd
, NULL_RTX
, 0);
4185 /* Emit the code to divide OP0 by OP1, putting the result in TARGET
4186 if that is convenient, and returning where the result is.
4187 You may request either the quotient or the remainder as the result;
4188 specify REM_FLAG nonzero to get the remainder.
4190 CODE is the expression code for which kind of division this is;
4191 it controls how rounding is done. MODE is the machine mode to use.
4192 UNSIGNEDP nonzero means do unsigned division. */
4194 /* ??? For CEIL_MOD_EXPR, can compute incorrect remainder with ANDI
4195 and then correct it by or'ing in missing high bits
4196 if result of ANDI is nonzero.
4197 For ROUND_MOD_EXPR, can use ANDI and then sign-extend the result.
4198 This could optimize to a bfexts instruction.
4199 But C doesn't use these operations, so their optimizations are
4201 /* ??? For modulo, we don't actually need the highpart of the first product,
4202 the low part will do nicely. And for small divisors, the second multiply
4203 can also be a low-part only multiply or even be completely left out.
4204 E.g. to calculate the remainder of a division by 3 with a 32 bit
4205 multiply, multiply with 0x55555556 and extract the upper two bits;
4206 the result is exact for inputs up to 0x1fffffff.
4207 The input range can be reduced by using cross-sum rules.
4208 For odd divisors >= 3, the following table gives right shift counts
4209 so that if a number is shifted by an integer multiple of the given
4210 amount, the remainder stays the same:
4211 2, 4, 3, 6, 10, 12, 4, 8, 18, 6, 11, 20, 18, 0, 5, 10, 12, 0, 12, 20,
4212 14, 12, 23, 21, 8, 0, 20, 18, 0, 0, 6, 12, 0, 22, 0, 18, 20, 30, 0, 0,
4213 0, 8, 0, 11, 12, 10, 36, 0, 30, 0, 0, 12, 0, 0, 0, 0, 44, 12, 24, 0,
4214 20, 0, 7, 14, 0, 18, 36, 0, 0, 46, 60, 0, 42, 0, 15, 24, 20, 0, 0, 33,
4215 0, 20, 0, 0, 18, 0, 60, 0, 0, 0, 0, 0, 40, 18, 0, 0, 12
4217 Cross-sum rules for even numbers can be derived by leaving as many bits
4218 to the right alone as the divisor has zeros to the right.
4219 E.g. if x is an unsigned 32 bit number:
4220 (x mod 12) == (((x & 1023) + ((x >> 8) & ~3)) * 0x15555558 >> 2 * 3) >> 28
4224 expand_divmod (int rem_flag
, enum tree_code code
, machine_mode mode
,
4225 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
4226 enum optab_methods methods
)
4228 machine_mode compute_mode
;
4230 rtx quotient
= 0, remainder
= 0;
4233 optab optab1
, optab2
;
4234 int op1_is_constant
, op1_is_pow2
= 0;
4235 int max_cost
, extra_cost
;
4236 static HOST_WIDE_INT last_div_const
= 0;
4237 bool speed
= optimize_insn_for_speed_p ();
4239 op1_is_constant
= CONST_INT_P (op1
);
4240 if (op1_is_constant
)
4242 wide_int ext_op1
= rtx_mode_t (op1
, mode
);
4243 op1_is_pow2
= (wi::popcount (ext_op1
) == 1
4245 && wi::popcount (wi::neg (ext_op1
)) == 1));
4249 This is the structure of expand_divmod:
4251 First comes code to fix up the operands so we can perform the operations
4252 correctly and efficiently.
4254 Second comes a switch statement with code specific for each rounding mode.
4255 For some special operands this code emits all RTL for the desired
4256 operation, for other cases, it generates only a quotient and stores it in
4257 QUOTIENT. The case for trunc division/remainder might leave quotient = 0,
4258 to indicate that it has not done anything.
4260 Last comes code that finishes the operation. If QUOTIENT is set and
4261 REM_FLAG is set, the remainder is computed as OP0 - QUOTIENT * OP1. If
4262 QUOTIENT is not set, it is computed using trunc rounding.
4264 We try to generate special code for division and remainder when OP1 is a
4265 constant. If |OP1| = 2**n we can use shifts and some other fast
4266 operations. For other values of OP1, we compute a carefully selected
4267 fixed-point approximation m = 1/OP1, and generate code that multiplies OP0
4270 In all cases but EXACT_DIV_EXPR, this multiplication requires the upper
4271 half of the product. Different strategies for generating the product are
4272 implemented in expmed_mult_highpart.
4274 If what we actually want is the remainder, we generate that by another
4275 by-constant multiplication and a subtraction. */
4277 /* We shouldn't be called with OP1 == const1_rtx, but some of the
4278 code below will malfunction if we are, so check here and handle
4279 the special case if so. */
4280 if (op1
== const1_rtx
)
4281 return rem_flag
? const0_rtx
: op0
;
4283 /* When dividing by -1, we could get an overflow.
4284 negv_optab can handle overflows. */
4285 if (! unsignedp
&& op1
== constm1_rtx
)
4289 return expand_unop (mode
, flag_trapv
&& GET_MODE_CLASS (mode
) == MODE_INT
4290 ? negv_optab
: neg_optab
, op0
, target
, 0);
4294 /* Don't use the function value register as a target
4295 since we have to read it as well as write it,
4296 and function-inlining gets confused by this. */
4297 && ((REG_P (target
) && REG_FUNCTION_VALUE_P (target
))
4298 /* Don't clobber an operand while doing a multi-step calculation. */
4299 || ((rem_flag
|| op1_is_constant
)
4300 && (reg_mentioned_p (target
, op0
)
4301 || (MEM_P (op0
) && MEM_P (target
))))
4302 || reg_mentioned_p (target
, op1
)
4303 || (MEM_P (op1
) && MEM_P (target
))))
4306 /* Get the mode in which to perform this computation. Normally it will
4307 be MODE, but sometimes we can't do the desired operation in MODE.
4308 If so, pick a wider mode in which we can do the operation. Convert
4309 to that mode at the start to avoid repeated conversions.
4311 First see what operations we need. These depend on the expression
4312 we are evaluating. (We assume that divxx3 insns exist under the
4313 same conditions that modxx3 insns and that these insns don't normally
4314 fail. If these assumptions are not correct, we may generate less
4315 efficient code in some cases.)
4317 Then see if we find a mode in which we can open-code that operation
4318 (either a division, modulus, or shift). Finally, check for the smallest
4319 mode for which we can do the operation with a library call. */
4321 /* We might want to refine this now that we have division-by-constant
4322 optimization. Since expmed_mult_highpart tries so many variants, it is
4323 not straightforward to generalize this. Maybe we should make an array
4324 of possible modes in init_expmed? Save this for GCC 2.7. */
4326 optab1
= (op1_is_pow2
4327 ? (unsignedp
? lshr_optab
: ashr_optab
)
4328 : (unsignedp
? udiv_optab
: sdiv_optab
));
4329 optab2
= (op1_is_pow2
? optab1
4330 : (unsignedp
? udivmod_optab
: sdivmod_optab
));
4332 if (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
)
4334 FOR_EACH_MODE_FROM (compute_mode
, mode
)
4335 if (optab_handler (optab1
, compute_mode
) != CODE_FOR_nothing
4336 || optab_handler (optab2
, compute_mode
) != CODE_FOR_nothing
)
4339 if (compute_mode
== VOIDmode
&& methods
== OPTAB_LIB_WIDEN
)
4340 FOR_EACH_MODE_FROM (compute_mode
, mode
)
4341 if (optab_libfunc (optab1
, compute_mode
)
4342 || optab_libfunc (optab2
, compute_mode
))
4346 compute_mode
= mode
;
4348 /* If we still couldn't find a mode, use MODE, but expand_binop will
4350 if (compute_mode
== VOIDmode
)
4351 compute_mode
= mode
;
4353 if (target
&& GET_MODE (target
) == compute_mode
)
4356 tquotient
= gen_reg_rtx (compute_mode
);
4359 /* It should be possible to restrict the precision to GET_MODE_BITSIZE
4360 (mode), and thereby get better code when OP1 is a constant. Do that
4361 later. It will require going over all usages of SIZE below. */
4362 size
= GET_MODE_BITSIZE (mode
);
4365 /* Only deduct something for a REM if the last divide done was
4366 for a different constant. Then set the constant of the last
4368 max_cost
= (unsignedp
4369 ? udiv_cost (speed
, compute_mode
)
4370 : sdiv_cost (speed
, compute_mode
));
4371 if (rem_flag
&& ! (last_div_const
!= 0 && op1_is_constant
4372 && INTVAL (op1
) == last_div_const
))
4373 max_cost
-= (mul_cost (speed
, compute_mode
)
4374 + add_cost (speed
, compute_mode
));
4376 last_div_const
= ! rem_flag
&& op1_is_constant
? INTVAL (op1
) : 0;
4378 /* Now convert to the best mode to use. */
4379 if (compute_mode
!= mode
)
4381 op0
= convert_modes (compute_mode
, mode
, op0
, unsignedp
);
4382 op1
= convert_modes (compute_mode
, mode
, op1
, unsignedp
);
4384 /* convert_modes may have placed op1 into a register, so we
4385 must recompute the following. */
4386 op1_is_constant
= CONST_INT_P (op1
);
4387 if (op1_is_constant
)
4389 wide_int ext_op1
= rtx_mode_t (op1
, compute_mode
);
4390 op1_is_pow2
= (wi::popcount (ext_op1
) == 1
4392 && wi::popcount (wi::neg (ext_op1
)) == 1));
4398 /* If one of the operands is a volatile MEM, copy it into a register. */
4400 if (MEM_P (op0
) && MEM_VOLATILE_P (op0
))
4401 op0
= force_reg (compute_mode
, op0
);
4402 if (MEM_P (op1
) && MEM_VOLATILE_P (op1
))
4403 op1
= force_reg (compute_mode
, op1
);
4405 /* If we need the remainder or if OP1 is constant, we need to
4406 put OP0 in a register in case it has any queued subexpressions. */
4407 if (rem_flag
|| op1_is_constant
)
4408 op0
= force_reg (compute_mode
, op0
);
4410 last
= get_last_insn ();
4412 /* Promote floor rounding to trunc rounding for unsigned operations. */
4415 if (code
== FLOOR_DIV_EXPR
)
4416 code
= TRUNC_DIV_EXPR
;
4417 if (code
== FLOOR_MOD_EXPR
)
4418 code
= TRUNC_MOD_EXPR
;
4419 if (code
== EXACT_DIV_EXPR
&& op1_is_pow2
)
4420 code
= TRUNC_DIV_EXPR
;
4423 if (op1
!= const0_rtx
)
4426 case TRUNC_MOD_EXPR
:
4427 case TRUNC_DIV_EXPR
:
4428 if (op1_is_constant
)
4430 scalar_int_mode int_mode
= as_a
<scalar_int_mode
> (compute_mode
);
4431 int size
= GET_MODE_BITSIZE (int_mode
);
4434 unsigned HOST_WIDE_INT mh
, ml
;
4435 int pre_shift
, post_shift
;
4437 wide_int wd
= rtx_mode_t (op1
, int_mode
);
4438 unsigned HOST_WIDE_INT d
= wd
.to_uhwi ();
4440 if (wi::popcount (wd
) == 1)
4442 pre_shift
= floor_log2 (d
);
4445 unsigned HOST_WIDE_INT mask
4446 = (HOST_WIDE_INT_1U
<< pre_shift
) - 1;
4448 = expand_binop (int_mode
, and_optab
, op0
,
4449 gen_int_mode (mask
, int_mode
),
4450 remainder
, 1, methods
);
4452 return gen_lowpart (mode
, remainder
);
4454 quotient
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
4455 pre_shift
, tquotient
, 1);
4457 else if (size
<= HOST_BITS_PER_WIDE_INT
)
4459 if (d
>= (HOST_WIDE_INT_1U
<< (size
- 1)))
4461 /* Most significant bit of divisor is set; emit an scc
4463 quotient
= emit_store_flag_force (tquotient
, GEU
, op0
, op1
,
4468 /* Find a suitable multiplier and right shift count
4469 instead of multiplying with D. */
4471 mh
= choose_multiplier (d
, size
, size
,
4472 &ml
, &post_shift
, &dummy
);
4474 /* If the suggested multiplier is more than SIZE bits,
4475 we can do better for even divisors, using an
4476 initial right shift. */
4477 if (mh
!= 0 && (d
& 1) == 0)
4479 pre_shift
= ctz_or_zero (d
);
4480 mh
= choose_multiplier (d
>> pre_shift
, size
,
4482 &ml
, &post_shift
, &dummy
);
4492 if (post_shift
- 1 >= BITS_PER_WORD
)
4496 = (shift_cost (speed
, int_mode
, post_shift
- 1)
4497 + shift_cost (speed
, int_mode
, 1)
4498 + 2 * add_cost (speed
, int_mode
));
4499 t1
= expmed_mult_highpart
4500 (int_mode
, op0
, gen_int_mode (ml
, int_mode
),
4501 NULL_RTX
, 1, max_cost
- extra_cost
);
4504 t2
= force_operand (gen_rtx_MINUS (int_mode
,
4507 t3
= expand_shift (RSHIFT_EXPR
, int_mode
,
4508 t2
, 1, NULL_RTX
, 1);
4509 t4
= force_operand (gen_rtx_PLUS (int_mode
,
4512 quotient
= expand_shift
4513 (RSHIFT_EXPR
, int_mode
, t4
,
4514 post_shift
- 1, tquotient
, 1);
4520 if (pre_shift
>= BITS_PER_WORD
4521 || post_shift
>= BITS_PER_WORD
)
4525 (RSHIFT_EXPR
, int_mode
, op0
,
4526 pre_shift
, NULL_RTX
, 1);
4528 = (shift_cost (speed
, int_mode
, pre_shift
)
4529 + shift_cost (speed
, int_mode
, post_shift
));
4530 t2
= expmed_mult_highpart
4532 gen_int_mode (ml
, int_mode
),
4533 NULL_RTX
, 1, max_cost
- extra_cost
);
4536 quotient
= expand_shift
4537 (RSHIFT_EXPR
, int_mode
, t2
,
4538 post_shift
, tquotient
, 1);
4542 else /* Too wide mode to use tricky code */
4545 insn
= get_last_insn ();
4547 set_dst_reg_note (insn
, REG_EQUAL
,
4548 gen_rtx_UDIV (int_mode
, op0
, op1
),
4551 else /* TRUNC_DIV, signed */
4553 unsigned HOST_WIDE_INT ml
;
4554 int lgup
, post_shift
;
4556 HOST_WIDE_INT d
= INTVAL (op1
);
4557 unsigned HOST_WIDE_INT abs_d
;
4559 /* Not prepared to handle division/remainder by
4560 0xffffffffffffffff8000000000000000 etc. */
4561 if (d
== HOST_WIDE_INT_MIN
&& size
> HOST_BITS_PER_WIDE_INT
)
4564 /* Since d might be INT_MIN, we have to cast to
4565 unsigned HOST_WIDE_INT before negating to avoid
4566 undefined signed overflow. */
4568 ? (unsigned HOST_WIDE_INT
) d
4569 : - (unsigned HOST_WIDE_INT
) d
);
4571 /* n rem d = n rem -d */
4572 if (rem_flag
&& d
< 0)
4575 op1
= gen_int_mode (abs_d
, int_mode
);
4581 quotient
= expand_unop (int_mode
, neg_optab
, op0
,
4583 else if (size
<= HOST_BITS_PER_WIDE_INT
4584 && abs_d
== HOST_WIDE_INT_1U
<< (size
- 1))
4586 /* This case is not handled correctly below. */
4587 quotient
= emit_store_flag (tquotient
, EQ
, op0
, op1
,
4592 else if (EXACT_POWER_OF_2_OR_ZERO_P (d
)
4593 && (size
<= HOST_BITS_PER_WIDE_INT
|| d
>= 0)
4595 ? smod_pow2_cheap (speed
, int_mode
)
4596 : sdiv_pow2_cheap (speed
, int_mode
))
4597 /* We assume that cheap metric is true if the
4598 optab has an expander for this mode. */
4599 && ((optab_handler ((rem_flag
? smod_optab
4602 != CODE_FOR_nothing
)
4603 || (optab_handler (sdivmod_optab
, int_mode
)
4604 != CODE_FOR_nothing
)))
4606 else if (EXACT_POWER_OF_2_OR_ZERO_P (abs_d
))
4610 remainder
= expand_smod_pow2 (int_mode
, op0
, d
);
4612 return gen_lowpart (mode
, remainder
);
4615 if (sdiv_pow2_cheap (speed
, int_mode
)
4616 && ((optab_handler (sdiv_optab
, int_mode
)
4617 != CODE_FOR_nothing
)
4618 || (optab_handler (sdivmod_optab
, int_mode
)
4619 != CODE_FOR_nothing
)))
4620 quotient
= expand_divmod (0, TRUNC_DIV_EXPR
,
4622 gen_int_mode (abs_d
,
4626 quotient
= expand_sdiv_pow2 (int_mode
, op0
, abs_d
);
4628 /* We have computed OP0 / abs(OP1). If OP1 is negative,
4629 negate the quotient. */
4632 insn
= get_last_insn ();
4634 && abs_d
< (HOST_WIDE_INT_1U
4635 << (HOST_BITS_PER_WIDE_INT
- 1)))
4636 set_dst_reg_note (insn
, REG_EQUAL
,
4637 gen_rtx_DIV (int_mode
, op0
,
4643 quotient
= expand_unop (int_mode
, neg_optab
,
4644 quotient
, quotient
, 0);
4647 else if (size
<= HOST_BITS_PER_WIDE_INT
)
4649 choose_multiplier (abs_d
, size
, size
- 1,
4650 &ml
, &post_shift
, &lgup
);
4651 if (ml
< HOST_WIDE_INT_1U
<< (size
- 1))
4655 if (post_shift
>= BITS_PER_WORD
4656 || size
- 1 >= BITS_PER_WORD
)
4659 extra_cost
= (shift_cost (speed
, int_mode
, post_shift
)
4660 + shift_cost (speed
, int_mode
, size
- 1)
4661 + add_cost (speed
, int_mode
));
4662 t1
= expmed_mult_highpart
4663 (int_mode
, op0
, gen_int_mode (ml
, int_mode
),
4664 NULL_RTX
, 0, max_cost
- extra_cost
);
4668 (RSHIFT_EXPR
, int_mode
, t1
,
4669 post_shift
, NULL_RTX
, 0);
4671 (RSHIFT_EXPR
, int_mode
, op0
,
4672 size
- 1, NULL_RTX
, 0);
4675 = force_operand (gen_rtx_MINUS (int_mode
, t3
, t2
),
4679 = force_operand (gen_rtx_MINUS (int_mode
, t2
, t3
),
4686 if (post_shift
>= BITS_PER_WORD
4687 || size
- 1 >= BITS_PER_WORD
)
4690 ml
|= HOST_WIDE_INT_M1U
<< (size
- 1);
4691 mlr
= gen_int_mode (ml
, int_mode
);
4692 extra_cost
= (shift_cost (speed
, int_mode
, post_shift
)
4693 + shift_cost (speed
, int_mode
, size
- 1)
4694 + 2 * add_cost (speed
, int_mode
));
4695 t1
= expmed_mult_highpart (int_mode
, op0
, mlr
,
4697 max_cost
- extra_cost
);
4700 t2
= force_operand (gen_rtx_PLUS (int_mode
, t1
, op0
),
4703 (RSHIFT_EXPR
, int_mode
, t2
,
4704 post_shift
, NULL_RTX
, 0);
4706 (RSHIFT_EXPR
, int_mode
, op0
,
4707 size
- 1, NULL_RTX
, 0);
4710 = force_operand (gen_rtx_MINUS (int_mode
, t4
, t3
),
4714 = force_operand (gen_rtx_MINUS (int_mode
, t3
, t4
),
4718 else /* Too wide mode to use tricky code */
4721 insn
= get_last_insn ();
4723 set_dst_reg_note (insn
, REG_EQUAL
,
4724 gen_rtx_DIV (int_mode
, op0
, op1
),
4730 delete_insns_since (last
);
4733 case FLOOR_DIV_EXPR
:
4734 case FLOOR_MOD_EXPR
:
4735 /* We will come here only for signed operations. */
4736 if (op1_is_constant
&& HWI_COMPUTABLE_MODE_P (compute_mode
))
4738 scalar_int_mode int_mode
= as_a
<scalar_int_mode
> (compute_mode
);
4739 int size
= GET_MODE_BITSIZE (int_mode
);
4740 unsigned HOST_WIDE_INT mh
, ml
;
4741 int pre_shift
, lgup
, post_shift
;
4742 HOST_WIDE_INT d
= INTVAL (op1
);
4746 /* We could just as easily deal with negative constants here,
4747 but it does not seem worth the trouble for GCC 2.6. */
4748 if (EXACT_POWER_OF_2_OR_ZERO_P (d
))
4750 pre_shift
= floor_log2 (d
);
4753 unsigned HOST_WIDE_INT mask
4754 = (HOST_WIDE_INT_1U
<< pre_shift
) - 1;
4755 remainder
= expand_binop
4756 (int_mode
, and_optab
, op0
,
4757 gen_int_mode (mask
, int_mode
),
4758 remainder
, 0, methods
);
4760 return gen_lowpart (mode
, remainder
);
4762 quotient
= expand_shift
4763 (RSHIFT_EXPR
, int_mode
, op0
,
4764 pre_shift
, tquotient
, 0);
4770 mh
= choose_multiplier (d
, size
, size
- 1,
4771 &ml
, &post_shift
, &lgup
);
4774 if (post_shift
< BITS_PER_WORD
4775 && size
- 1 < BITS_PER_WORD
)
4778 (RSHIFT_EXPR
, int_mode
, op0
,
4779 size
- 1, NULL_RTX
, 0);
4780 t2
= expand_binop (int_mode
, xor_optab
, op0
, t1
,
4781 NULL_RTX
, 0, OPTAB_WIDEN
);
4782 extra_cost
= (shift_cost (speed
, int_mode
, post_shift
)
4783 + shift_cost (speed
, int_mode
, size
- 1)
4784 + 2 * add_cost (speed
, int_mode
));
4785 t3
= expmed_mult_highpart
4786 (int_mode
, t2
, gen_int_mode (ml
, int_mode
),
4787 NULL_RTX
, 1, max_cost
- extra_cost
);
4791 (RSHIFT_EXPR
, int_mode
, t3
,
4792 post_shift
, NULL_RTX
, 1);
4793 quotient
= expand_binop (int_mode
, xor_optab
,
4794 t4
, t1
, tquotient
, 0,
4802 rtx nsign
, t1
, t2
, t3
, t4
;
4803 t1
= force_operand (gen_rtx_PLUS (int_mode
,
4804 op0
, constm1_rtx
), NULL_RTX
);
4805 t2
= expand_binop (int_mode
, ior_optab
, op0
, t1
, NULL_RTX
,
4807 nsign
= expand_shift (RSHIFT_EXPR
, int_mode
, t2
,
4808 size
- 1, NULL_RTX
, 0);
4809 t3
= force_operand (gen_rtx_MINUS (int_mode
, t1
, nsign
),
4811 t4
= expand_divmod (0, TRUNC_DIV_EXPR
, int_mode
, t3
, op1
,
4816 t5
= expand_unop (int_mode
, one_cmpl_optab
, nsign
,
4818 quotient
= force_operand (gen_rtx_PLUS (int_mode
, t4
, t5
),
4826 delete_insns_since (last
);
4828 /* Try using an instruction that produces both the quotient and
4829 remainder, using truncation. We can easily compensate the quotient
4830 or remainder to get floor rounding, once we have the remainder.
4831 Notice that we compute also the final remainder value here,
4832 and return the result right away. */
4833 if (target
== 0 || GET_MODE (target
) != compute_mode
)
4834 target
= gen_reg_rtx (compute_mode
);
4839 = REG_P (target
) ? target
: gen_reg_rtx (compute_mode
);
4840 quotient
= gen_reg_rtx (compute_mode
);
4845 = REG_P (target
) ? target
: gen_reg_rtx (compute_mode
);
4846 remainder
= gen_reg_rtx (compute_mode
);
4849 if (expand_twoval_binop (sdivmod_optab
, op0
, op1
,
4850 quotient
, remainder
, 0))
4852 /* This could be computed with a branch-less sequence.
4853 Save that for later. */
4855 rtx_code_label
*label
= gen_label_rtx ();
4856 do_cmp_and_jump (remainder
, const0_rtx
, EQ
, compute_mode
, label
);
4857 tem
= expand_binop (compute_mode
, xor_optab
, op0
, op1
,
4858 NULL_RTX
, 0, OPTAB_WIDEN
);
4859 do_cmp_and_jump (tem
, const0_rtx
, GE
, compute_mode
, label
);
4860 expand_dec (quotient
, const1_rtx
);
4861 expand_inc (remainder
, op1
);
4863 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
4866 /* No luck with division elimination or divmod. Have to do it
4867 by conditionally adjusting op0 *and* the result. */
4869 rtx_code_label
*label1
, *label2
, *label3
, *label4
, *label5
;
4873 quotient
= gen_reg_rtx (compute_mode
);
4874 adjusted_op0
= copy_to_mode_reg (compute_mode
, op0
);
4875 label1
= gen_label_rtx ();
4876 label2
= gen_label_rtx ();
4877 label3
= gen_label_rtx ();
4878 label4
= gen_label_rtx ();
4879 label5
= gen_label_rtx ();
4880 do_cmp_and_jump (op1
, const0_rtx
, LT
, compute_mode
, label2
);
4881 do_cmp_and_jump (adjusted_op0
, const0_rtx
, LT
, compute_mode
, label1
);
4882 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4883 quotient
, 0, methods
);
4884 if (tem
!= quotient
)
4885 emit_move_insn (quotient
, tem
);
4886 emit_jump_insn (targetm
.gen_jump (label5
));
4888 emit_label (label1
);
4889 expand_inc (adjusted_op0
, const1_rtx
);
4890 emit_jump_insn (targetm
.gen_jump (label4
));
4892 emit_label (label2
);
4893 do_cmp_and_jump (adjusted_op0
, const0_rtx
, GT
, compute_mode
, label3
);
4894 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4895 quotient
, 0, methods
);
4896 if (tem
!= quotient
)
4897 emit_move_insn (quotient
, tem
);
4898 emit_jump_insn (targetm
.gen_jump (label5
));
4900 emit_label (label3
);
4901 expand_dec (adjusted_op0
, const1_rtx
);
4902 emit_label (label4
);
4903 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4904 quotient
, 0, methods
);
4905 if (tem
!= quotient
)
4906 emit_move_insn (quotient
, tem
);
4907 expand_dec (quotient
, const1_rtx
);
4908 emit_label (label5
);
4917 && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1
))
4918 && (HWI_COMPUTABLE_MODE_P (compute_mode
)
4919 || INTVAL (op1
) >= 0))
4921 scalar_int_mode int_mode
4922 = as_a
<scalar_int_mode
> (compute_mode
);
4924 unsigned HOST_WIDE_INT d
= INTVAL (op1
);
4925 t1
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
4926 floor_log2 (d
), tquotient
, 1);
4927 t2
= expand_binop (int_mode
, and_optab
, op0
,
4928 gen_int_mode (d
- 1, int_mode
),
4929 NULL_RTX
, 1, methods
);
4930 t3
= gen_reg_rtx (int_mode
);
4931 t3
= emit_store_flag (t3
, NE
, t2
, const0_rtx
, int_mode
, 1, 1);
4934 rtx_code_label
*lab
;
4935 lab
= gen_label_rtx ();
4936 do_cmp_and_jump (t2
, const0_rtx
, EQ
, int_mode
, lab
);
4937 expand_inc (t1
, const1_rtx
);
4942 quotient
= force_operand (gen_rtx_PLUS (int_mode
, t1
, t3
),
4947 /* Try using an instruction that produces both the quotient and
4948 remainder, using truncation. We can easily compensate the
4949 quotient or remainder to get ceiling rounding, once we have the
4950 remainder. Notice that we compute also the final remainder
4951 value here, and return the result right away. */
4952 if (target
== 0 || GET_MODE (target
) != compute_mode
)
4953 target
= gen_reg_rtx (compute_mode
);
4957 remainder
= (REG_P (target
)
4958 ? target
: gen_reg_rtx (compute_mode
));
4959 quotient
= gen_reg_rtx (compute_mode
);
4963 quotient
= (REG_P (target
)
4964 ? target
: gen_reg_rtx (compute_mode
));
4965 remainder
= gen_reg_rtx (compute_mode
);
4968 if (expand_twoval_binop (udivmod_optab
, op0
, op1
, quotient
,
4971 /* This could be computed with a branch-less sequence.
4972 Save that for later. */
4973 rtx_code_label
*label
= gen_label_rtx ();
4974 do_cmp_and_jump (remainder
, const0_rtx
, EQ
,
4975 compute_mode
, label
);
4976 expand_inc (quotient
, const1_rtx
);
4977 expand_dec (remainder
, op1
);
4979 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
4982 /* No luck with division elimination or divmod. Have to do it
4983 by conditionally adjusting op0 *and* the result. */
4985 rtx_code_label
*label1
, *label2
;
4986 rtx adjusted_op0
, tem
;
4988 quotient
= gen_reg_rtx (compute_mode
);
4989 adjusted_op0
= copy_to_mode_reg (compute_mode
, op0
);
4990 label1
= gen_label_rtx ();
4991 label2
= gen_label_rtx ();
4992 do_cmp_and_jump (adjusted_op0
, const0_rtx
, NE
,
4993 compute_mode
, label1
);
4994 emit_move_insn (quotient
, const0_rtx
);
4995 emit_jump_insn (targetm
.gen_jump (label2
));
4997 emit_label (label1
);
4998 expand_dec (adjusted_op0
, const1_rtx
);
4999 tem
= expand_binop (compute_mode
, udiv_optab
, adjusted_op0
, op1
,
5000 quotient
, 1, methods
);
5001 if (tem
!= quotient
)
5002 emit_move_insn (quotient
, tem
);
5003 expand_inc (quotient
, const1_rtx
);
5004 emit_label (label2
);
5009 if (op1_is_constant
&& EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1
))
5010 && INTVAL (op1
) >= 0)
5012 /* This is extremely similar to the code for the unsigned case
5013 above. For 2.7 we should merge these variants, but for
5014 2.6.1 I don't want to touch the code for unsigned since that
5015 get used in C. The signed case will only be used by other
5019 unsigned HOST_WIDE_INT d
= INTVAL (op1
);
5020 t1
= expand_shift (RSHIFT_EXPR
, compute_mode
, op0
,
5021 floor_log2 (d
), tquotient
, 0);
5022 t2
= expand_binop (compute_mode
, and_optab
, op0
,
5023 gen_int_mode (d
- 1, compute_mode
),
5024 NULL_RTX
, 1, methods
);
5025 t3
= gen_reg_rtx (compute_mode
);
5026 t3
= emit_store_flag (t3
, NE
, t2
, const0_rtx
,
5027 compute_mode
, 1, 1);
5030 rtx_code_label
*lab
;
5031 lab
= gen_label_rtx ();
5032 do_cmp_and_jump (t2
, const0_rtx
, EQ
, compute_mode
, lab
);
5033 expand_inc (t1
, const1_rtx
);
5038 quotient
= force_operand (gen_rtx_PLUS (compute_mode
,
5044 /* Try using an instruction that produces both the quotient and
5045 remainder, using truncation. We can easily compensate the
5046 quotient or remainder to get ceiling rounding, once we have the
5047 remainder. Notice that we compute also the final remainder
5048 value here, and return the result right away. */
5049 if (target
== 0 || GET_MODE (target
) != compute_mode
)
5050 target
= gen_reg_rtx (compute_mode
);
5053 remainder
= (REG_P (target
)
5054 ? target
: gen_reg_rtx (compute_mode
));
5055 quotient
= gen_reg_rtx (compute_mode
);
5059 quotient
= (REG_P (target
)
5060 ? target
: gen_reg_rtx (compute_mode
));
5061 remainder
= gen_reg_rtx (compute_mode
);
5064 if (expand_twoval_binop (sdivmod_optab
, op0
, op1
, quotient
,
5067 /* This could be computed with a branch-less sequence.
5068 Save that for later. */
5070 rtx_code_label
*label
= gen_label_rtx ();
5071 do_cmp_and_jump (remainder
, const0_rtx
, EQ
,
5072 compute_mode
, label
);
5073 tem
= expand_binop (compute_mode
, xor_optab
, op0
, op1
,
5074 NULL_RTX
, 0, OPTAB_WIDEN
);
5075 do_cmp_and_jump (tem
, const0_rtx
, LT
, compute_mode
, label
);
5076 expand_inc (quotient
, const1_rtx
);
5077 expand_dec (remainder
, op1
);
5079 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
5082 /* No luck with division elimination or divmod. Have to do it
5083 by conditionally adjusting op0 *and* the result. */
5085 rtx_code_label
*label1
, *label2
, *label3
, *label4
, *label5
;
5089 quotient
= gen_reg_rtx (compute_mode
);
5090 adjusted_op0
= copy_to_mode_reg (compute_mode
, op0
);
5091 label1
= gen_label_rtx ();
5092 label2
= gen_label_rtx ();
5093 label3
= gen_label_rtx ();
5094 label4
= gen_label_rtx ();
5095 label5
= gen_label_rtx ();
5096 do_cmp_and_jump (op1
, const0_rtx
, LT
, compute_mode
, label2
);
5097 do_cmp_and_jump (adjusted_op0
, const0_rtx
, GT
,
5098 compute_mode
, label1
);
5099 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
5100 quotient
, 0, methods
);
5101 if (tem
!= quotient
)
5102 emit_move_insn (quotient
, tem
);
5103 emit_jump_insn (targetm
.gen_jump (label5
));
5105 emit_label (label1
);
5106 expand_dec (adjusted_op0
, const1_rtx
);
5107 emit_jump_insn (targetm
.gen_jump (label4
));
5109 emit_label (label2
);
5110 do_cmp_and_jump (adjusted_op0
, const0_rtx
, LT
,
5111 compute_mode
, label3
);
5112 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
5113 quotient
, 0, methods
);
5114 if (tem
!= quotient
)
5115 emit_move_insn (quotient
, tem
);
5116 emit_jump_insn (targetm
.gen_jump (label5
));
5118 emit_label (label3
);
5119 expand_inc (adjusted_op0
, const1_rtx
);
5120 emit_label (label4
);
5121 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
5122 quotient
, 0, methods
);
5123 if (tem
!= quotient
)
5124 emit_move_insn (quotient
, tem
);
5125 expand_inc (quotient
, const1_rtx
);
5126 emit_label (label5
);
5131 case EXACT_DIV_EXPR
:
5132 if (op1_is_constant
&& HWI_COMPUTABLE_MODE_P (compute_mode
))
5134 scalar_int_mode int_mode
= as_a
<scalar_int_mode
> (compute_mode
);
5135 int size
= GET_MODE_BITSIZE (int_mode
);
5136 HOST_WIDE_INT d
= INTVAL (op1
);
5137 unsigned HOST_WIDE_INT ml
;
5141 pre_shift
= ctz_or_zero (d
);
5142 ml
= invert_mod2n (d
>> pre_shift
, size
);
5143 t1
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
5144 pre_shift
, NULL_RTX
, unsignedp
);
5145 quotient
= expand_mult (int_mode
, t1
, gen_int_mode (ml
, int_mode
),
5148 insn
= get_last_insn ();
5149 set_dst_reg_note (insn
, REG_EQUAL
,
5150 gen_rtx_fmt_ee (unsignedp
? UDIV
: DIV
,
5151 int_mode
, op0
, op1
),
5156 case ROUND_DIV_EXPR
:
5157 case ROUND_MOD_EXPR
:
5160 scalar_int_mode int_mode
= as_a
<scalar_int_mode
> (compute_mode
);
5162 rtx_code_label
*label
;
5163 label
= gen_label_rtx ();
5164 quotient
= gen_reg_rtx (int_mode
);
5165 remainder
= gen_reg_rtx (int_mode
);
5166 if (expand_twoval_binop (udivmod_optab
, op0
, op1
, quotient
, remainder
, 1) == 0)
5169 quotient
= expand_binop (int_mode
, udiv_optab
, op0
, op1
,
5170 quotient
, 1, methods
);
5171 tem
= expand_mult (int_mode
, quotient
, op1
, NULL_RTX
, 1);
5172 remainder
= expand_binop (int_mode
, sub_optab
, op0
, tem
,
5173 remainder
, 1, methods
);
5175 tem
= plus_constant (int_mode
, op1
, -1);
5176 tem
= expand_shift (RSHIFT_EXPR
, int_mode
, tem
, 1, NULL_RTX
, 1);
5177 do_cmp_and_jump (remainder
, tem
, LEU
, int_mode
, label
);
5178 expand_inc (quotient
, const1_rtx
);
5179 expand_dec (remainder
, op1
);
5184 scalar_int_mode int_mode
= as_a
<scalar_int_mode
> (compute_mode
);
5185 int size
= GET_MODE_BITSIZE (int_mode
);
5186 rtx abs_rem
, abs_op1
, tem
, mask
;
5187 rtx_code_label
*label
;
5188 label
= gen_label_rtx ();
5189 quotient
= gen_reg_rtx (int_mode
);
5190 remainder
= gen_reg_rtx (int_mode
);
5191 if (expand_twoval_binop (sdivmod_optab
, op0
, op1
, quotient
, remainder
, 0) == 0)
5194 quotient
= expand_binop (int_mode
, sdiv_optab
, op0
, op1
,
5195 quotient
, 0, methods
);
5196 tem
= expand_mult (int_mode
, quotient
, op1
, NULL_RTX
, 0);
5197 remainder
= expand_binop (int_mode
, sub_optab
, op0
, tem
,
5198 remainder
, 0, methods
);
5200 abs_rem
= expand_abs (int_mode
, remainder
, NULL_RTX
, 1, 0);
5201 abs_op1
= expand_abs (int_mode
, op1
, NULL_RTX
, 1, 0);
5202 tem
= expand_shift (LSHIFT_EXPR
, int_mode
, abs_rem
,
5204 do_cmp_and_jump (tem
, abs_op1
, LTU
, int_mode
, label
);
5205 tem
= expand_binop (int_mode
, xor_optab
, op0
, op1
,
5206 NULL_RTX
, 0, OPTAB_WIDEN
);
5207 mask
= expand_shift (RSHIFT_EXPR
, int_mode
, tem
,
5208 size
- 1, NULL_RTX
, 0);
5209 tem
= expand_binop (int_mode
, xor_optab
, mask
, const1_rtx
,
5210 NULL_RTX
, 0, OPTAB_WIDEN
);
5211 tem
= expand_binop (int_mode
, sub_optab
, tem
, mask
,
5212 NULL_RTX
, 0, OPTAB_WIDEN
);
5213 expand_inc (quotient
, tem
);
5214 tem
= expand_binop (int_mode
, xor_optab
, mask
, op1
,
5215 NULL_RTX
, 0, OPTAB_WIDEN
);
5216 tem
= expand_binop (int_mode
, sub_optab
, tem
, mask
,
5217 NULL_RTX
, 0, OPTAB_WIDEN
);
5218 expand_dec (remainder
, tem
);
5221 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
5229 if (target
&& GET_MODE (target
) != compute_mode
)
5234 /* Try to produce the remainder without producing the quotient.
5235 If we seem to have a divmod pattern that does not require widening,
5236 don't try widening here. We should really have a WIDEN argument
5237 to expand_twoval_binop, since what we'd really like to do here is
5238 1) try a mod insn in compute_mode
5239 2) try a divmod insn in compute_mode
5240 3) try a div insn in compute_mode and multiply-subtract to get
5242 4) try the same things with widening allowed. */
5244 = sign_expand_binop (compute_mode
, umod_optab
, smod_optab
,
5247 ((optab_handler (optab2
, compute_mode
)
5248 != CODE_FOR_nothing
)
5249 ? OPTAB_DIRECT
: OPTAB_WIDEN
));
5252 /* No luck there. Can we do remainder and divide at once
5253 without a library call? */
5254 remainder
= gen_reg_rtx (compute_mode
);
5255 if (! expand_twoval_binop ((unsignedp
5259 NULL_RTX
, remainder
, unsignedp
))
5264 return gen_lowpart (mode
, remainder
);
5267 /* Produce the quotient. Try a quotient insn, but not a library call.
5268 If we have a divmod in this mode, use it in preference to widening
5269 the div (for this test we assume it will not fail). Note that optab2
5270 is set to the one of the two optabs that the call below will use. */
5272 = sign_expand_binop (compute_mode
, udiv_optab
, sdiv_optab
,
5273 op0
, op1
, rem_flag
? NULL_RTX
: target
,
5275 ((optab_handler (optab2
, compute_mode
)
5276 != CODE_FOR_nothing
)
5277 ? OPTAB_DIRECT
: OPTAB_WIDEN
));
5281 /* No luck there. Try a quotient-and-remainder insn,
5282 keeping the quotient alone. */
5283 quotient
= gen_reg_rtx (compute_mode
);
5284 if (! expand_twoval_binop (unsignedp
? udivmod_optab
: sdivmod_optab
,
5286 quotient
, NULL_RTX
, unsignedp
))
5290 /* Still no luck. If we are not computing the remainder,
5291 use a library call for the quotient. */
5292 quotient
= sign_expand_binop (compute_mode
,
5293 udiv_optab
, sdiv_optab
,
5295 unsignedp
, methods
);
5302 if (target
&& GET_MODE (target
) != compute_mode
)
5307 /* No divide instruction either. Use library for remainder. */
5308 remainder
= sign_expand_binop (compute_mode
, umod_optab
, smod_optab
,
5310 unsignedp
, methods
);
5311 /* No remainder function. Try a quotient-and-remainder
5312 function, keeping the remainder. */
5314 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
5316 remainder
= gen_reg_rtx (compute_mode
);
5317 if (!expand_twoval_binop_libfunc
5318 (unsignedp
? udivmod_optab
: sdivmod_optab
,
5320 NULL_RTX
, remainder
,
5321 unsignedp
? UMOD
: MOD
))
5322 remainder
= NULL_RTX
;
5327 /* We divided. Now finish doing X - Y * (X / Y). */
5328 remainder
= expand_mult (compute_mode
, quotient
, op1
,
5329 NULL_RTX
, unsignedp
);
5330 remainder
= expand_binop (compute_mode
, sub_optab
, op0
,
5331 remainder
, target
, unsignedp
,
5336 if (methods
!= OPTAB_LIB_WIDEN
5337 && (rem_flag
? remainder
: quotient
) == NULL_RTX
)
5340 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
5343 /* Return a tree node with data type TYPE, describing the value of X.
5344 Usually this is an VAR_DECL, if there is no obvious better choice.
5345 X may be an expression, however we only support those expressions
5346 generated by loop.c. */
5349 make_tree (tree type
, rtx x
)
5353 switch (GET_CODE (x
))
5356 case CONST_WIDE_INT
:
5357 t
= wide_int_to_tree (type
, rtx_mode_t (x
, TYPE_MODE (type
)));
5361 STATIC_ASSERT (HOST_BITS_PER_WIDE_INT
* 2 <= MAX_BITSIZE_MODE_ANY_INT
);
5362 if (TARGET_SUPPORTS_WIDE_INT
== 0 && GET_MODE (x
) == VOIDmode
)
5363 t
= wide_int_to_tree (type
,
5364 wide_int::from_array (&CONST_DOUBLE_LOW (x
), 2,
5365 HOST_BITS_PER_WIDE_INT
* 2));
5367 t
= build_real (type
, *CONST_DOUBLE_REAL_VALUE (x
));
5373 unsigned int npatterns
= CONST_VECTOR_NPATTERNS (x
);
5374 unsigned int nelts_per_pattern
= CONST_VECTOR_NELTS_PER_PATTERN (x
);
5375 tree itype
= TREE_TYPE (type
);
5377 /* Build a tree with vector elements. */
5378 tree_vector_builder
elts (type
, npatterns
, nelts_per_pattern
);
5379 unsigned int count
= elts
.encoded_nelts ();
5380 for (unsigned int i
= 0; i
< count
; ++i
)
5382 rtx elt
= CONST_VECTOR_ELT (x
, i
);
5383 elts
.quick_push (make_tree (itype
, elt
));
5386 return elts
.build ();
5390 return fold_build2 (PLUS_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5391 make_tree (type
, XEXP (x
, 1)));
5394 return fold_build2 (MINUS_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5395 make_tree (type
, XEXP (x
, 1)));
5398 return fold_build1 (NEGATE_EXPR
, type
, make_tree (type
, XEXP (x
, 0)));
5401 return fold_build2 (MULT_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5402 make_tree (type
, XEXP (x
, 1)));
5405 return fold_build2 (LSHIFT_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5406 make_tree (type
, XEXP (x
, 1)));
5409 t
= unsigned_type_for (type
);
5410 return fold_convert (type
, build2 (RSHIFT_EXPR
, t
,
5411 make_tree (t
, XEXP (x
, 0)),
5412 make_tree (type
, XEXP (x
, 1))));
5415 t
= signed_type_for (type
);
5416 return fold_convert (type
, build2 (RSHIFT_EXPR
, t
,
5417 make_tree (t
, XEXP (x
, 0)),
5418 make_tree (type
, XEXP (x
, 1))));
5421 if (TREE_CODE (type
) != REAL_TYPE
)
5422 t
= signed_type_for (type
);
5426 return fold_convert (type
, build2 (TRUNC_DIV_EXPR
, t
,
5427 make_tree (t
, XEXP (x
, 0)),
5428 make_tree (t
, XEXP (x
, 1))));
5430 t
= unsigned_type_for (type
);
5431 return fold_convert (type
, build2 (TRUNC_DIV_EXPR
, t
,
5432 make_tree (t
, XEXP (x
, 0)),
5433 make_tree (t
, XEXP (x
, 1))));
5437 t
= lang_hooks
.types
.type_for_mode (GET_MODE (XEXP (x
, 0)),
5438 GET_CODE (x
) == ZERO_EXTEND
);
5439 return fold_convert (type
, make_tree (t
, XEXP (x
, 0)));
5442 return make_tree (type
, XEXP (x
, 0));
5445 t
= SYMBOL_REF_DECL (x
);
5447 return fold_convert (type
, build_fold_addr_expr (t
));
5451 if (CONST_POLY_INT_P (x
))
5452 return wide_int_to_tree (t
, const_poly_int_value (x
));
5454 t
= build_decl (RTL_LOCATION (x
), VAR_DECL
, NULL_TREE
, type
);
5456 /* If TYPE is a POINTER_TYPE, we might need to convert X from
5457 address mode to pointer mode. */
5458 if (POINTER_TYPE_P (type
))
5459 x
= convert_memory_address_addr_space
5460 (SCALAR_INT_TYPE_MODE (type
), x
, TYPE_ADDR_SPACE (TREE_TYPE (type
)));
5462 /* Note that we do *not* use SET_DECL_RTL here, because we do not
5463 want set_decl_rtl to go adjusting REG_ATTRS for this temporary. */
5464 t
->decl_with_rtl
.rtl
= x
;
5470 /* Compute the logical-and of OP0 and OP1, storing it in TARGET
5471 and returning TARGET.
5473 If TARGET is 0, a pseudo-register or constant is returned. */
5476 expand_and (machine_mode mode
, rtx op0
, rtx op1
, rtx target
)
5480 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
5481 tem
= simplify_binary_operation (AND
, mode
, op0
, op1
);
5483 tem
= expand_binop (mode
, and_optab
, op0
, op1
, target
, 0, OPTAB_LIB_WIDEN
);
5487 else if (tem
!= target
)
5488 emit_move_insn (target
, tem
);
5492 /* Helper function for emit_store_flag. */
5494 emit_cstore (rtx target
, enum insn_code icode
, enum rtx_code code
,
5495 machine_mode mode
, machine_mode compare_mode
,
5496 int unsignedp
, rtx x
, rtx y
, int normalizep
,
5497 machine_mode target_mode
)
5499 class expand_operand ops
[4];
5500 rtx op0
, comparison
, subtarget
;
5502 scalar_int_mode result_mode
= targetm
.cstore_mode (icode
);
5503 scalar_int_mode int_target_mode
;
5505 last
= get_last_insn ();
5506 x
= prepare_operand (icode
, x
, 2, mode
, compare_mode
, unsignedp
);
5507 y
= prepare_operand (icode
, y
, 3, mode
, compare_mode
, unsignedp
);
5510 delete_insns_since (last
);
5514 if (target_mode
== VOIDmode
)
5515 int_target_mode
= result_mode
;
5517 int_target_mode
= as_a
<scalar_int_mode
> (target_mode
);
5519 target
= gen_reg_rtx (int_target_mode
);
5521 comparison
= gen_rtx_fmt_ee (code
, result_mode
, x
, y
);
5523 create_output_operand (&ops
[0], optimize
? NULL_RTX
: target
, result_mode
);
5524 create_fixed_operand (&ops
[1], comparison
);
5525 create_fixed_operand (&ops
[2], x
);
5526 create_fixed_operand (&ops
[3], y
);
5527 if (!maybe_expand_insn (icode
, 4, ops
))
5529 delete_insns_since (last
);
5532 subtarget
= ops
[0].value
;
5534 /* If we are converting to a wider mode, first convert to
5535 INT_TARGET_MODE, then normalize. This produces better combining
5536 opportunities on machines that have a SIGN_EXTRACT when we are
5537 testing a single bit. This mostly benefits the 68k.
5539 If STORE_FLAG_VALUE does not have the sign bit set when
5540 interpreted in MODE, we can do this conversion as unsigned, which
5541 is usually more efficient. */
5542 if (GET_MODE_PRECISION (int_target_mode
) > GET_MODE_PRECISION (result_mode
))
5544 gcc_assert (GET_MODE_PRECISION (result_mode
) != 1
5545 || STORE_FLAG_VALUE
== 1 || STORE_FLAG_VALUE
== -1);
5547 bool unsignedp
= (STORE_FLAG_VALUE
>= 0);
5548 convert_move (target
, subtarget
, unsignedp
);
5551 result_mode
= int_target_mode
;
5556 /* If we want to keep subexpressions around, don't reuse our last
5561 /* Now normalize to the proper value in MODE. Sometimes we don't
5562 have to do anything. */
5563 if (normalizep
== 0 || normalizep
== STORE_FLAG_VALUE
)
5565 /* STORE_FLAG_VALUE might be the most negative number, so write
5566 the comparison this way to avoid a compiler-time warning. */
5567 else if (- normalizep
== STORE_FLAG_VALUE
)
5568 op0
= expand_unop (result_mode
, neg_optab
, op0
, subtarget
, 0);
5570 /* We don't want to use STORE_FLAG_VALUE < 0 below since this makes
5571 it hard to use a value of just the sign bit due to ANSI integer
5572 constant typing rules. */
5573 else if (val_signbit_known_set_p (result_mode
, STORE_FLAG_VALUE
))
5574 op0
= expand_shift (RSHIFT_EXPR
, result_mode
, op0
,
5575 GET_MODE_BITSIZE (result_mode
) - 1, subtarget
,
5579 gcc_assert (STORE_FLAG_VALUE
& 1);
5581 op0
= expand_and (result_mode
, op0
, const1_rtx
, subtarget
);
5582 if (normalizep
== -1)
5583 op0
= expand_unop (result_mode
, neg_optab
, op0
, op0
, 0);
5586 /* If we were converting to a smaller mode, do the conversion now. */
5587 if (int_target_mode
!= result_mode
)
5589 convert_move (target
, op0
, 0);
5597 /* A subroutine of emit_store_flag only including "tricks" that do not
5598 need a recursive call. These are kept separate to avoid infinite
5602 emit_store_flag_1 (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5603 machine_mode mode
, int unsignedp
, int normalizep
,
5604 machine_mode target_mode
)
5607 enum insn_code icode
;
5608 machine_mode compare_mode
;
5609 enum mode_class mclass
;
5610 enum rtx_code scode
;
5613 code
= unsigned_condition (code
);
5614 scode
= swap_condition (code
);
5616 /* If one operand is constant, make it the second one. Only do this
5617 if the other operand is not constant as well. */
5619 if (swap_commutative_operands_p (op0
, op1
))
5621 std::swap (op0
, op1
);
5622 code
= swap_condition (code
);
5625 if (mode
== VOIDmode
)
5626 mode
= GET_MODE (op0
);
5628 if (CONST_SCALAR_INT_P (op1
))
5629 canonicalize_comparison (mode
, &code
, &op1
);
5631 /* For some comparisons with 1 and -1, we can convert this to
5632 comparisons with zero. This will often produce more opportunities for
5633 store-flag insns. */
5638 if (op1
== const1_rtx
)
5639 op1
= const0_rtx
, code
= LE
;
5642 if (op1
== constm1_rtx
)
5643 op1
= const0_rtx
, code
= LT
;
5646 if (op1
== const1_rtx
)
5647 op1
= const0_rtx
, code
= GT
;
5650 if (op1
== constm1_rtx
)
5651 op1
= const0_rtx
, code
= GE
;
5654 if (op1
== const1_rtx
)
5655 op1
= const0_rtx
, code
= NE
;
5658 if (op1
== const1_rtx
)
5659 op1
= const0_rtx
, code
= EQ
;
5665 /* If this is A < 0 or A >= 0, we can do this by taking the ones
5666 complement of A (for GE) and shifting the sign bit to the low bit. */
5667 scalar_int_mode int_mode
;
5668 if (op1
== const0_rtx
&& (code
== LT
|| code
== GE
)
5669 && is_int_mode (mode
, &int_mode
)
5670 && (normalizep
|| STORE_FLAG_VALUE
== 1
5671 || val_signbit_p (int_mode
, STORE_FLAG_VALUE
)))
5673 scalar_int_mode int_target_mode
;
5677 int_target_mode
= int_mode
;
5680 /* If the result is to be wider than OP0, it is best to convert it
5681 first. If it is to be narrower, it is *incorrect* to convert it
5683 int_target_mode
= as_a
<scalar_int_mode
> (target_mode
);
5684 if (GET_MODE_SIZE (int_target_mode
) > GET_MODE_SIZE (int_mode
))
5686 op0
= convert_modes (int_target_mode
, int_mode
, op0
, 0);
5687 int_mode
= int_target_mode
;
5691 if (int_target_mode
!= int_mode
)
5695 op0
= expand_unop (int_mode
, one_cmpl_optab
, op0
,
5696 ((STORE_FLAG_VALUE
== 1 || normalizep
)
5697 ? 0 : subtarget
), 0);
5699 if (STORE_FLAG_VALUE
== 1 || normalizep
)
5700 /* If we are supposed to produce a 0/1 value, we want to do
5701 a logical shift from the sign bit to the low-order bit; for
5702 a -1/0 value, we do an arithmetic shift. */
5703 op0
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
5704 GET_MODE_BITSIZE (int_mode
) - 1,
5705 subtarget
, normalizep
!= -1);
5707 if (int_mode
!= int_target_mode
)
5708 op0
= convert_modes (int_target_mode
, int_mode
, op0
, 0);
5713 /* Next try expanding this via the backend's cstore<mode>4. */
5714 mclass
= GET_MODE_CLASS (mode
);
5715 FOR_EACH_MODE_FROM (compare_mode
, mode
)
5717 machine_mode optab_mode
= mclass
== MODE_CC
? CCmode
: compare_mode
;
5718 icode
= optab_handler (cstore_optab
, optab_mode
);
5719 if (icode
!= CODE_FOR_nothing
)
5721 do_pending_stack_adjust ();
5722 rtx tem
= emit_cstore (target
, icode
, code
, mode
, compare_mode
,
5723 unsignedp
, op0
, op1
, normalizep
, target_mode
);
5727 if (GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5729 tem
= emit_cstore (target
, icode
, scode
, mode
, compare_mode
,
5730 unsignedp
, op1
, op0
, normalizep
, target_mode
);
5738 /* If we are comparing a double-word integer with zero or -1, we can
5739 convert the comparison into one involving a single word. */
5740 if (is_int_mode (mode
, &int_mode
)
5741 && GET_MODE_BITSIZE (int_mode
) == BITS_PER_WORD
* 2
5742 && (!MEM_P (op0
) || ! MEM_VOLATILE_P (op0
)))
5745 if ((code
== EQ
|| code
== NE
)
5746 && (op1
== const0_rtx
|| op1
== constm1_rtx
))
5750 /* Do a logical OR or AND of the two words and compare the
5752 op00
= simplify_gen_subreg (word_mode
, op0
, int_mode
, 0);
5753 op01
= simplify_gen_subreg (word_mode
, op0
, int_mode
, UNITS_PER_WORD
);
5754 tem
= expand_binop (word_mode
,
5755 op1
== const0_rtx
? ior_optab
: and_optab
,
5756 op00
, op01
, NULL_RTX
, unsignedp
,
5760 tem
= emit_store_flag (NULL_RTX
, code
, tem
, op1
, word_mode
,
5761 unsignedp
, normalizep
);
5763 else if ((code
== LT
|| code
== GE
) && op1
== const0_rtx
)
5767 /* If testing the sign bit, can just test on high word. */
5768 op0h
= simplify_gen_subreg (word_mode
, op0
, int_mode
,
5769 subreg_highpart_offset (word_mode
,
5771 tem
= emit_store_flag (NULL_RTX
, code
, op0h
, op1
, word_mode
,
5772 unsignedp
, normalizep
);
5779 if (target_mode
== VOIDmode
|| GET_MODE (tem
) == target_mode
)
5782 target
= gen_reg_rtx (target_mode
);
5784 convert_move (target
, tem
,
5785 !val_signbit_known_set_p (word_mode
,
5786 (normalizep
? normalizep
5787 : STORE_FLAG_VALUE
)));
5795 /* Subroutine of emit_store_flag that handles cases in which the operands
5796 are scalar integers. SUBTARGET is the target to use for temporary
5797 operations and TRUEVAL is the value to store when the condition is
5798 true. All other arguments are as for emit_store_flag. */
5801 emit_store_flag_int (rtx target
, rtx subtarget
, enum rtx_code code
, rtx op0
,
5802 rtx op1
, scalar_int_mode mode
, int unsignedp
,
5803 int normalizep
, rtx trueval
)
5805 machine_mode target_mode
= target
? GET_MODE (target
) : VOIDmode
;
5806 rtx_insn
*last
= get_last_insn ();
5808 /* If this is an equality comparison of integers, we can try to exclusive-or
5809 (or subtract) the two operands and use a recursive call to try the
5810 comparison with zero. Don't do any of these cases if branches are
5813 if ((code
== EQ
|| code
== NE
) && op1
!= const0_rtx
)
5815 rtx tem
= expand_binop (mode
, xor_optab
, op0
, op1
, subtarget
, 1,
5819 tem
= expand_binop (mode
, sub_optab
, op0
, op1
, subtarget
, 1,
5822 tem
= emit_store_flag (target
, code
, tem
, const0_rtx
,
5823 mode
, unsignedp
, normalizep
);
5827 delete_insns_since (last
);
5830 /* For integer comparisons, try the reverse comparison. However, for
5831 small X and if we'd have anyway to extend, implementing "X != 0"
5832 as "-(int)X >> 31" is still cheaper than inverting "(int)X == 0". */
5833 rtx_code rcode
= reverse_condition (code
);
5834 if (can_compare_p (rcode
, mode
, ccp_store_flag
)
5835 && ! (optab_handler (cstore_optab
, mode
) == CODE_FOR_nothing
5837 && GET_MODE_SIZE (mode
) < UNITS_PER_WORD
5838 && op1
== const0_rtx
))
5840 int want_add
= ((STORE_FLAG_VALUE
== 1 && normalizep
== -1)
5841 || (STORE_FLAG_VALUE
== -1 && normalizep
== 1));
5843 /* Again, for the reverse comparison, use either an addition or a XOR. */
5845 && rtx_cost (GEN_INT (normalizep
), mode
, PLUS
, 1,
5846 optimize_insn_for_speed_p ()) == 0)
5848 rtx tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
5849 STORE_FLAG_VALUE
, target_mode
);
5851 tem
= expand_binop (target_mode
, add_optab
, tem
,
5852 gen_int_mode (normalizep
, target_mode
),
5853 target
, 0, OPTAB_WIDEN
);
5858 && rtx_cost (trueval
, mode
, XOR
, 1,
5859 optimize_insn_for_speed_p ()) == 0)
5861 rtx tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
5862 normalizep
, target_mode
);
5864 tem
= expand_binop (target_mode
, xor_optab
, tem
, trueval
, target
,
5865 INTVAL (trueval
) >= 0, OPTAB_WIDEN
);
5870 delete_insns_since (last
);
5873 /* Some other cases we can do are EQ, NE, LE, and GT comparisons with
5874 the constant zero. Reject all other comparisons at this point. Only
5875 do LE and GT if branches are expensive since they are expensive on
5876 2-operand machines. */
5878 if (op1
!= const0_rtx
5879 || (code
!= EQ
&& code
!= NE
5880 && (BRANCH_COST (optimize_insn_for_speed_p (),
5881 false) <= 1 || (code
!= LE
&& code
!= GT
))))
5884 /* Try to put the result of the comparison in the sign bit. Assume we can't
5885 do the necessary operation below. */
5889 /* To see if A <= 0, compute (A | (A - 1)). A <= 0 iff that result has
5890 the sign bit set. */
5894 /* This is destructive, so SUBTARGET can't be OP0. */
5895 if (rtx_equal_p (subtarget
, op0
))
5898 tem
= expand_binop (mode
, sub_optab
, op0
, const1_rtx
, subtarget
, 0,
5901 tem
= expand_binop (mode
, ior_optab
, op0
, tem
, subtarget
, 0,
5905 /* To see if A > 0, compute (((signed) A) << BITS) - A, where BITS is the
5906 number of bits in the mode of OP0, minus one. */
5910 if (rtx_equal_p (subtarget
, op0
))
5913 tem
= maybe_expand_shift (RSHIFT_EXPR
, mode
, op0
,
5914 GET_MODE_BITSIZE (mode
) - 1,
5917 tem
= expand_binop (mode
, sub_optab
, tem
, op0
, subtarget
, 0,
5921 if (code
== EQ
|| code
== NE
)
5923 /* For EQ or NE, one way to do the comparison is to apply an operation
5924 that converts the operand into a positive number if it is nonzero
5925 or zero if it was originally zero. Then, for EQ, we subtract 1 and
5926 for NE we negate. This puts the result in the sign bit. Then we
5927 normalize with a shift, if needed.
5929 Two operations that can do the above actions are ABS and FFS, so try
5930 them. If that doesn't work, and MODE is smaller than a full word,
5931 we can use zero-extension to the wider mode (an unsigned conversion)
5932 as the operation. */
5934 /* Note that ABS doesn't yield a positive number for INT_MIN, but
5935 that is compensated by the subsequent overflow when subtracting
5938 if (optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)
5939 tem
= expand_unop (mode
, abs_optab
, op0
, subtarget
, 1);
5940 else if (optab_handler (ffs_optab
, mode
) != CODE_FOR_nothing
)
5941 tem
= expand_unop (mode
, ffs_optab
, op0
, subtarget
, 1);
5942 else if (GET_MODE_SIZE (mode
) < UNITS_PER_WORD
)
5944 tem
= convert_modes (word_mode
, mode
, op0
, 1);
5951 tem
= expand_binop (mode
, sub_optab
, tem
, const1_rtx
, subtarget
,
5954 tem
= expand_unop (mode
, neg_optab
, tem
, subtarget
, 0);
5957 /* If we couldn't do it that way, for NE we can "or" the two's complement
5958 of the value with itself. For EQ, we take the one's complement of
5959 that "or", which is an extra insn, so we only handle EQ if branches
5964 || BRANCH_COST (optimize_insn_for_speed_p (),
5967 if (rtx_equal_p (subtarget
, op0
))
5970 tem
= expand_unop (mode
, neg_optab
, op0
, subtarget
, 0);
5971 tem
= expand_binop (mode
, ior_optab
, tem
, op0
, subtarget
, 0,
5974 if (tem
&& code
== EQ
)
5975 tem
= expand_unop (mode
, one_cmpl_optab
, tem
, subtarget
, 0);
5979 if (tem
&& normalizep
)
5980 tem
= maybe_expand_shift (RSHIFT_EXPR
, mode
, tem
,
5981 GET_MODE_BITSIZE (mode
) - 1,
5982 subtarget
, normalizep
== 1);
5988 else if (GET_MODE (tem
) != target_mode
)
5990 convert_move (target
, tem
, 0);
5993 else if (!subtarget
)
5995 emit_move_insn (target
, tem
);
6000 delete_insns_since (last
);
6005 /* Emit a store-flags instruction for comparison CODE on OP0 and OP1
6006 and storing in TARGET. Normally return TARGET.
6007 Return 0 if that cannot be done.
6009 MODE is the mode to use for OP0 and OP1 should they be CONST_INTs. If
6010 it is VOIDmode, they cannot both be CONST_INT.
6012 UNSIGNEDP is for the case where we have to widen the operands
6013 to perform the operation. It says to use zero-extension.
6015 NORMALIZEP is 1 if we should convert the result to be either zero
6016 or one. Normalize is -1 if we should convert the result to be
6017 either zero or -1. If NORMALIZEP is zero, the result will be left
6018 "raw" out of the scc insn. */
6021 emit_store_flag (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
6022 machine_mode mode
, int unsignedp
, int normalizep
)
6024 machine_mode target_mode
= target
? GET_MODE (target
) : VOIDmode
;
6025 enum rtx_code rcode
;
6030 /* If we compare constants, we shouldn't use a store-flag operation,
6031 but a constant load. We can get there via the vanilla route that
6032 usually generates a compare-branch sequence, but will in this case
6033 fold the comparison to a constant, and thus elide the branch. */
6034 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
6037 tem
= emit_store_flag_1 (target
, code
, op0
, op1
, mode
, unsignedp
, normalizep
,
6042 /* If we reached here, we can't do this with a scc insn, however there
6043 are some comparisons that can be done in other ways. Don't do any
6044 of these cases if branches are very cheap. */
6045 if (BRANCH_COST (optimize_insn_for_speed_p (), false) == 0)
6048 /* See what we need to return. We can only return a 1, -1, or the
6051 if (normalizep
== 0)
6053 if (STORE_FLAG_VALUE
== 1 || STORE_FLAG_VALUE
== -1)
6054 normalizep
= STORE_FLAG_VALUE
;
6056 else if (val_signbit_p (mode
, STORE_FLAG_VALUE
))
6062 last
= get_last_insn ();
6064 /* If optimizing, use different pseudo registers for each insn, instead
6065 of reusing the same pseudo. This leads to better CSE, but slows
6066 down the compiler, since there are more pseudos. */
6067 subtarget
= (!optimize
6068 && (target_mode
== mode
)) ? target
: NULL_RTX
;
6069 trueval
= GEN_INT (normalizep
? normalizep
: STORE_FLAG_VALUE
);
6071 /* For floating-point comparisons, try the reverse comparison or try
6072 changing the "orderedness" of the comparison. */
6073 if (GET_MODE_CLASS (mode
) == MODE_FLOAT
)
6075 enum rtx_code first_code
;
6078 rcode
= reverse_condition_maybe_unordered (code
);
6079 if (can_compare_p (rcode
, mode
, ccp_store_flag
)
6080 && (code
== ORDERED
|| code
== UNORDERED
6081 || (! HONOR_NANS (mode
) && (code
== LTGT
|| code
== UNEQ
))
6082 || (! HONOR_SNANS (mode
) && (code
== EQ
|| code
== NE
))))
6084 int want_add
= ((STORE_FLAG_VALUE
== 1 && normalizep
== -1)
6085 || (STORE_FLAG_VALUE
== -1 && normalizep
== 1));
6087 /* For the reverse comparison, use either an addition or a XOR. */
6089 && rtx_cost (GEN_INT (normalizep
), mode
, PLUS
, 1,
6090 optimize_insn_for_speed_p ()) == 0)
6092 tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
6093 STORE_FLAG_VALUE
, target_mode
);
6095 return expand_binop (target_mode
, add_optab
, tem
,
6096 gen_int_mode (normalizep
, target_mode
),
6097 target
, 0, OPTAB_WIDEN
);
6100 && rtx_cost (trueval
, mode
, XOR
, 1,
6101 optimize_insn_for_speed_p ()) == 0)
6103 tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
6104 normalizep
, target_mode
);
6106 return expand_binop (target_mode
, xor_optab
, tem
, trueval
,
6107 target
, INTVAL (trueval
) >= 0,
6112 delete_insns_since (last
);
6114 /* Cannot split ORDERED and UNORDERED, only try the above trick. */
6115 if (code
== ORDERED
|| code
== UNORDERED
)
6118 and_them
= split_comparison (code
, mode
, &first_code
, &code
);
6120 /* If there are no NaNs, the first comparison should always fall through.
6121 Effectively change the comparison to the other one. */
6122 if (!HONOR_NANS (mode
))
6124 gcc_assert (first_code
== (and_them
? ORDERED
: UNORDERED
));
6125 return emit_store_flag_1 (target
, code
, op0
, op1
, mode
, 0, normalizep
,
6129 if (!HAVE_conditional_move
)
6132 /* Do not turn a trapping comparison into a non-trapping one. */
6133 if ((code
!= EQ
&& code
!= NE
&& code
!= UNEQ
&& code
!= LTGT
)
6134 && flag_trapping_math
)
6137 /* Try using a setcc instruction for ORDERED/UNORDERED, followed by a
6138 conditional move. */
6139 tem
= emit_store_flag_1 (subtarget
, first_code
, op0
, op1
, mode
, 0,
6140 normalizep
, target_mode
);
6145 tem
= emit_conditional_move (target
, { code
, op0
, op1
, mode
},
6146 tem
, const0_rtx
, GET_MODE (tem
), 0);
6148 tem
= emit_conditional_move (target
, { code
, op0
, op1
, mode
},
6149 trueval
, tem
, GET_MODE (tem
), 0);
6152 delete_insns_since (last
);
6156 /* The remaining tricks only apply to integer comparisons. */
6158 scalar_int_mode int_mode
;
6159 if (is_int_mode (mode
, &int_mode
))
6160 return emit_store_flag_int (target
, subtarget
, code
, op0
, op1
, int_mode
,
6161 unsignedp
, normalizep
, trueval
);
6166 /* Like emit_store_flag, but always succeeds. */
6169 emit_store_flag_force (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
6170 machine_mode mode
, int unsignedp
, int normalizep
)
6173 rtx_code_label
*label
;
6174 rtx trueval
, falseval
;
6176 /* First see if emit_store_flag can do the job. */
6177 tem
= emit_store_flag (target
, code
, op0
, op1
, mode
, unsignedp
, normalizep
);
6181 /* If one operand is constant, make it the second one. Only do this
6182 if the other operand is not constant as well. */
6183 if (swap_commutative_operands_p (op0
, op1
))
6185 std::swap (op0
, op1
);
6186 code
= swap_condition (code
);
6189 if (mode
== VOIDmode
)
6190 mode
= GET_MODE (op0
);
6193 target
= gen_reg_rtx (word_mode
);
6195 /* If this failed, we have to do this with set/compare/jump/set code.
6196 For foo != 0, if foo is in OP0, just replace it with 1 if nonzero. */
6197 trueval
= normalizep
? GEN_INT (normalizep
) : const1_rtx
;
6199 && GET_MODE_CLASS (mode
) == MODE_INT
6202 && op1
== const0_rtx
)
6204 label
= gen_label_rtx ();
6205 do_compare_rtx_and_jump (target
, const0_rtx
, EQ
, unsignedp
, mode
,
6206 NULL_RTX
, NULL
, label
,
6207 profile_probability::uninitialized ());
6208 emit_move_insn (target
, trueval
);
6214 || reg_mentioned_p (target
, op0
) || reg_mentioned_p (target
, op1
))
6215 target
= gen_reg_rtx (GET_MODE (target
));
6217 /* Jump in the right direction if the target cannot implement CODE
6218 but can jump on its reverse condition. */
6219 falseval
= const0_rtx
;
6220 if (! can_compare_p (code
, mode
, ccp_jump
)
6221 && (! FLOAT_MODE_P (mode
)
6222 || code
== ORDERED
|| code
== UNORDERED
6223 || (! HONOR_NANS (mode
) && (code
== LTGT
|| code
== UNEQ
))
6224 || (! HONOR_SNANS (mode
) && (code
== EQ
|| code
== NE
))))
6226 enum rtx_code rcode
;
6227 if (FLOAT_MODE_P (mode
))
6228 rcode
= reverse_condition_maybe_unordered (code
);
6230 rcode
= reverse_condition (code
);
6232 /* Canonicalize to UNORDERED for the libcall. */
6233 if (can_compare_p (rcode
, mode
, ccp_jump
)
6234 || (code
== ORDERED
&& ! can_compare_p (ORDERED
, mode
, ccp_jump
)))
6237 trueval
= const0_rtx
;
6242 emit_move_insn (target
, trueval
);
6243 label
= gen_label_rtx ();
6244 do_compare_rtx_and_jump (op0
, op1
, code
, unsignedp
, mode
, NULL_RTX
, NULL
,
6245 label
, profile_probability::uninitialized ());
6247 emit_move_insn (target
, falseval
);
6253 /* Helper function for canonicalize_cmp_for_target. Swap between inclusive
6254 and exclusive ranges in order to create an equivalent comparison. See
6255 canonicalize_cmp_for_target for the possible cases. */
6257 static enum rtx_code
6258 equivalent_cmp_code (enum rtx_code code
)
6284 /* Choose the more appropiate immediate in scalar integer comparisons. The
6285 purpose of this is to end up with an immediate which can be loaded into a
6286 register in fewer moves, if possible.
6288 For each integer comparison there exists an equivalent choice:
6289 i) a > b or a >= b + 1
6290 ii) a <= b or a < b + 1
6291 iii) a >= b or a > b - 1
6292 iv) a < b or a <= b - 1
6294 MODE is the mode of the first operand.
6295 CODE points to the comparison code.
6296 IMM points to the rtx containing the immediate. *IMM must satisfy
6297 CONST_SCALAR_INT_P on entry and continues to satisfy CONST_SCALAR_INT_P
6301 canonicalize_comparison (machine_mode mode
, enum rtx_code
*code
, rtx
*imm
)
6303 if (!SCALAR_INT_MODE_P (mode
))
6307 enum signop sgn
= unsigned_condition_p (*code
) ? UNSIGNED
: SIGNED
;
6309 /* Extract the immediate value from the rtx. */
6310 wide_int imm_val
= rtx_mode_t (*imm
, mode
);
6312 if (*code
== GT
|| *code
== GTU
|| *code
== LE
|| *code
== LEU
)
6314 else if (*code
== GE
|| *code
== GEU
|| *code
== LT
|| *code
== LTU
)
6319 /* Check for overflow/underflow in the case of signed values and
6320 wrapping around in the case of unsigned values. If any occur
6321 cancel the optimization. */
6322 wi::overflow_type overflow
= wi::OVF_NONE
;
6326 imm_modif
= wi::add (imm_val
, 1, sgn
, &overflow
);
6328 imm_modif
= wi::sub (imm_val
, 1, sgn
, &overflow
);
6333 /* The following creates a pseudo; if we cannot do that, bail out. */
6334 if (!can_create_pseudo_p ())
6337 rtx reg
= gen_rtx_REG (mode
, LAST_VIRTUAL_REGISTER
+ 1);
6338 rtx new_imm
= immed_wide_int_const (imm_modif
, mode
);
6340 rtx_insn
*old_rtx
= gen_move_insn (reg
, *imm
);
6341 rtx_insn
*new_rtx
= gen_move_insn (reg
, new_imm
);
6343 /* Update the immediate and the code. */
6344 if (insn_cost (old_rtx
, true) > insn_cost (new_rtx
, true))
6346 *code
= equivalent_cmp_code (*code
);
6353 /* Perform possibly multi-word comparison and conditional jump to LABEL
6354 if ARG1 OP ARG2 true where ARG1 and ARG2 are of mode MODE. This is
6355 now a thin wrapper around do_compare_rtx_and_jump. */
6358 do_cmp_and_jump (rtx arg1
, rtx arg2
, enum rtx_code op
, machine_mode mode
,
6359 rtx_code_label
*label
)
6361 int unsignedp
= (op
== LTU
|| op
== LEU
|| op
== GTU
|| op
== GEU
);
6362 do_compare_rtx_and_jump (arg1
, arg2
, op
, unsignedp
, mode
, NULL_RTX
,
6363 NULL
, label
, profile_probability::uninitialized ());