gcc/config/riscv/autovec.md

   1 ;; Machine description for auto-vectorization using RVV for GNU compiler.
   2 ;; Copyright (C) 2023 Free Software Foundation, Inc.
   3 ;; Contributed by Juzhe Zhong (juzhe.zhong@rivai.ai), RiVAI Technologies Ltd.
   4
   5 ;; This file is part of GCC.
   6
   7 ;; GCC is free software; you can redistribute it and/or modify
   8 ;; it under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11
  12 ;; GCC is distributed in the hope that it will be useful,
  13 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;; GNU General Public License for more details.
  16
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; =========================================================================
  22 ;; == Loads/Stores
  23 ;; =========================================================================
  24
  25 (define_expand "mask_len_load<mode><vm>"
  26   [(match_operand:V 0 "register_operand")
  27    (match_operand:V 1 "memory_operand")
  28    (match_operand:<VM> 2 "vector_mask_operand")
  29    (match_operand 3 "autovec_length_operand")
  30    (match_operand 4 "const_0_operand")]
  31   "TARGET_VECTOR"
  32 {
  33   riscv_vector::expand_load_store (operands, true);
  34   DONE;
  35 })
  36
  37 (define_expand "mask_len_store<mode><vm>"
  38   [(match_operand:V 0 "memory_operand")
  39    (match_operand:V 1 "register_operand")
  40    (match_operand:<VM> 2 "vector_mask_operand")
  41    (match_operand 3 "autovec_length_operand")
  42    (match_operand 4 "const_0_operand")]
  43   "TARGET_VECTOR"
  44 {
  45   riscv_vector::expand_load_store (operands, false);
  46   DONE;
  47 })
  48
  49 ;; =========================================================================
  50 ;; == Gather Load
  51 ;; =========================================================================
  52
  53 (define_expand "mask_len_gather_load<RATIO64:mode><RATIO64I:mode>"
  54   [(match_operand:RATIO64 0 "register_operand")
  55    (match_operand 1 "pmode_reg_or_0_operand")
  56    (match_operand:RATIO64I 2 "register_operand")
  57    (match_operand 3 "<RATIO64:gs_extension>")
  58    (match_operand 4 "<RATIO64:gs_scale>")
  59    (match_operand:<RATIO64:VM> 5 "vector_mask_operand")
  60    (match_operand 6 "autovec_length_operand")
  61    (match_operand 7 "const_0_operand")]
  62   "TARGET_VECTOR"
  63 {
  64   riscv_vector::expand_gather_scatter (operands, true);
  65   DONE;
  66 })
  67
  68 (define_expand "mask_len_gather_load<RATIO32:mode><RATIO32I:mode>"
  69   [(match_operand:RATIO32 0 "register_operand")
  70    (match_operand 1 "pmode_reg_or_0_operand")
  71    (match_operand:RATIO32I 2 "register_operand")
  72    (match_operand 3 "<RATIO32:gs_extension>")
  73    (match_operand 4 "<RATIO32:gs_scale>")
  74    (match_operand:<RATIO32:VM> 5 "vector_mask_operand")
  75    (match_operand 6 "autovec_length_operand")
  76    (match_operand 7 "const_0_operand")]
  77   "TARGET_VECTOR"
  78 {
  79   riscv_vector::expand_gather_scatter (operands, true);
  80   DONE;
  81 })
  82
  83 (define_expand "mask_len_gather_load<RATIO16:mode><RATIO16I:mode>"
  84   [(match_operand:RATIO16 0 "register_operand")
  85    (match_operand 1 "pmode_reg_or_0_operand")
  86    (match_operand:RATIO16I 2 "register_operand")
  87    (match_operand 3 "<RATIO16:gs_extension>")
  88    (match_operand 4 "<RATIO16:gs_scale>")
  89    (match_operand:<RATIO16:VM> 5 "vector_mask_operand")
  90    (match_operand 6 "autovec_length_operand")
  91    (match_operand 7 "const_0_operand")]
  92   "TARGET_VECTOR"
  93 {
  94   riscv_vector::expand_gather_scatter (operands, true);
  95   DONE;
  96 })
  97
  98 (define_expand "mask_len_gather_load<RATIO8:mode><RATIO8I:mode>"
  99   [(match_operand:RATIO8 0 "register_operand")
 100    (match_operand 1 "pmode_reg_or_0_operand")
 101    (match_operand:RATIO8I 2 "register_operand")
 102    (match_operand 3 "<RATIO8:gs_extension>")
 103    (match_operand 4 "<RATIO8:gs_scale>")
 104    (match_operand:<RATIO8:VM> 5 "vector_mask_operand")
 105    (match_operand 6 "autovec_length_operand")
 106    (match_operand 7 "const_0_operand")]
 107   "TARGET_VECTOR"
 108 {
 109   riscv_vector::expand_gather_scatter (operands, true);
 110   DONE;
 111 })
 112
 113 (define_expand "mask_len_gather_load<RATIO4:mode><RATIO4I:mode>"
 114   [(match_operand:RATIO4 0 "register_operand")
 115    (match_operand 1 "pmode_reg_or_0_operand")
 116    (match_operand:RATIO4I 2 "register_operand")
 117    (match_operand 3 "<RATIO4:gs_extension>")
 118    (match_operand 4 "<RATIO4:gs_scale>")
 119    (match_operand:<RATIO4:VM> 5 "vector_mask_operand")
 120    (match_operand 6 "autovec_length_operand")
 121    (match_operand 7 "const_0_operand")]
 122   "TARGET_VECTOR"
 123 {
 124   riscv_vector::expand_gather_scatter (operands, true);
 125   DONE;
 126 })
 127
 128 (define_expand "mask_len_gather_load<RATIO2:mode><RATIO2I:mode>"
 129   [(match_operand:RATIO2 0 "register_operand")
 130    (match_operand 1 "pmode_reg_or_0_operand")
 131    (match_operand:RATIO2I 2 "register_operand")
 132    (match_operand 3 "<RATIO2:gs_extension>")
 133    (match_operand 4 "<RATIO2:gs_scale>")
 134    (match_operand:<RATIO2:VM> 5 "vector_mask_operand")
 135    (match_operand 6 "autovec_length_operand")
 136    (match_operand 7 "const_0_operand")]
 137   "TARGET_VECTOR"
 138 {
 139   riscv_vector::expand_gather_scatter (operands, true);
 140   DONE;
 141 })
 142
 143 ;; When SEW = 8 and LMUL = 8, we can't find any index mode with
 144 ;; larger SEW. Since RVV indexed load/store support zero extend
 145 ;; implicitly and not support scaling, we should only allow
 146 ;; operands[3] and operands[4] to be const_1_operand.
 147 (define_expand "mask_len_gather_load<RATIO1:mode><RATIO1:mode>"
 148   [(match_operand:RATIO1 0 "register_operand")
 149    (match_operand 1 "pmode_reg_or_0_operand")
 150    (match_operand:RATIO1 2 "register_operand")
 151    (match_operand 3 "<RATIO1:gs_extension>")
 152    (match_operand 4 "<RATIO1:gs_scale>")
 153    (match_operand:<RATIO1:VM> 5 "vector_mask_operand")
 154    (match_operand 6 "autovec_length_operand")
 155    (match_operand 7 "const_0_operand")]
 156   "TARGET_VECTOR"
 157 {
 158   riscv_vector::expand_gather_scatter (operands, true);
 159   DONE;
 160 })
 161
 162 ;; =========================================================================
 163 ;; == Scatter Store
 164 ;; =========================================================================
 165
 166 (define_expand "mask_len_scatter_store<RATIO64:mode><RATIO64I:mode>"
 167   [(match_operand 0 "pmode_reg_or_0_operand")
 168    (match_operand:RATIO64I 1 "register_operand")
 169    (match_operand 2 "<RATIO64:gs_extension>")
 170    (match_operand 3 "<RATIO64:gs_scale>")
 171    (match_operand:RATIO64 4 "register_operand")
 172    (match_operand:<RATIO64:VM> 5 "vector_mask_operand")
 173    (match_operand 6 "autovec_length_operand")
 174    (match_operand 7 "const_0_operand")]
 175   "TARGET_VECTOR"
 176 {
 177   riscv_vector::expand_gather_scatter (operands, false);
 178   DONE;
 179 })
 180
 181 (define_expand "mask_len_scatter_store<RATIO32:mode><RATIO32I:mode>"
 182   [(match_operand 0 "pmode_reg_or_0_operand")
 183    (match_operand:RATIO32I 1 "register_operand")
 184    (match_operand 2 "<RATIO32:gs_extension>")
 185    (match_operand 3 "<RATIO32:gs_scale>")
 186    (match_operand:RATIO32 4 "register_operand")
 187    (match_operand:<RATIO32:VM> 5 "vector_mask_operand")
 188    (match_operand 6 "autovec_length_operand")
 189    (match_operand 7 "const_0_operand")]
 190   "TARGET_VECTOR"
 191 {
 192   riscv_vector::expand_gather_scatter (operands, false);
 193   DONE;
 194 })
 195
 196 (define_expand "mask_len_scatter_store<RATIO16:mode><RATIO16I:mode>"
 197   [(match_operand 0 "pmode_reg_or_0_operand")
 198    (match_operand:RATIO16I 1 "register_operand")
 199    (match_operand 2 "<RATIO16:gs_extension>")
 200    (match_operand 3 "<RATIO16:gs_scale>")
 201    (match_operand:RATIO16 4 "register_operand")
 202    (match_operand:<RATIO16:VM> 5 "vector_mask_operand")
 203    (match_operand 6 "autovec_length_operand")
 204    (match_operand 7 "const_0_operand")]
 205   "TARGET_VECTOR"
 206 {
 207   riscv_vector::expand_gather_scatter (operands, false);
 208   DONE;
 209 })
 210
 211 (define_expand "mask_len_scatter_store<RATIO8:mode><RATIO8I:mode>"
 212   [(match_operand 0 "pmode_reg_or_0_operand")
 213    (match_operand:RATIO8I 1 "register_operand")
 214    (match_operand 2 "<RATIO8:gs_extension>")
 215    (match_operand 3 "<RATIO8:gs_scale>")
 216    (match_operand:RATIO8 4 "register_operand")
 217    (match_operand:<RATIO8:VM> 5 "vector_mask_operand")
 218    (match_operand 6 "autovec_length_operand")
 219    (match_operand 7 "const_0_operand")]
 220   "TARGET_VECTOR"
 221 {
 222   riscv_vector::expand_gather_scatter (operands, false);
 223   DONE;
 224 })
 225
 226 (define_expand "mask_len_scatter_store<RATIO4:mode><RATIO4I:mode>"
 227   [(match_operand 0 "pmode_reg_or_0_operand")
 228    (match_operand:RATIO4I 1 "register_operand")
 229    (match_operand 2 "<RATIO4:gs_extension>")
 230    (match_operand 3 "<RATIO4:gs_scale>")
 231    (match_operand:RATIO4 4 "register_operand")
 232    (match_operand:<RATIO4:VM> 5 "vector_mask_operand")
 233    (match_operand 6 "autovec_length_operand")
 234    (match_operand 7 "const_0_operand")]
 235   "TARGET_VECTOR"
 236 {
 237   riscv_vector::expand_gather_scatter (operands, false);
 238   DONE;
 239 })
 240
 241 (define_expand "mask_len_scatter_store<RATIO2:mode><RATIO2I:mode>"
 242   [(match_operand 0 "pmode_reg_or_0_operand")
 243    (match_operand:RATIO2I 1 "register_operand")
 244    (match_operand 2 "<RATIO2:gs_extension>")
 245    (match_operand 3 "<RATIO2:gs_scale>")
 246    (match_operand:RATIO2 4 "register_operand")
 247    (match_operand:<RATIO2:VM> 5 "vector_mask_operand")
 248    (match_operand 6 "autovec_length_operand")
 249    (match_operand 7 "const_0_operand")]
 250   "TARGET_VECTOR"
 251 {
 252   riscv_vector::expand_gather_scatter (operands, false);
 253   DONE;
 254 })
 255
 256 ;; When SEW = 8 and LMUL = 8, we can't find any index mode with
 257 ;; larger SEW. Since RVV indexed load/store support zero extend
 258 ;; implicitly and not support scaling, we should only allow
 259 ;; operands[3] and operands[4] to be const_1_operand.
 260 (define_expand "mask_len_scatter_store<RATIO1:mode><RATIO1:mode>"
 261   [(match_operand 0 "pmode_reg_or_0_operand")
 262    (match_operand:RATIO1 1 "register_operand")
 263    (match_operand 2 "<RATIO1:gs_extension>")
 264    (match_operand 3 "<RATIO1:gs_scale>")
 265    (match_operand:RATIO1 4 "register_operand")
 266    (match_operand:<RATIO1:VM> 5 "vector_mask_operand")
 267    (match_operand 6 "autovec_length_operand")
 268    (match_operand 7 "const_0_operand")]
 269   "TARGET_VECTOR"
 270 {
 271   riscv_vector::expand_gather_scatter (operands, false);
 272   DONE;
 273 })
 274
 275 ;; =========================================================================
 276 ;; == Array Load/Store
 277 ;; =========================================================================
 278
 279 (define_expand "vec_mask_len_load_lanes<mode><vsingle>"
 280   [(match_operand:VT 0 "register_operand")
 281    (match_operand:VT 1 "memory_operand")
 282    (match_operand:<VM> 2 "vector_mask_operand")
 283    (match_operand 3 "autovec_length_operand")
 284    (match_operand 4 "const_0_operand")]
 285   "TARGET_VECTOR"
 286   {
 287     riscv_vector::expand_lanes_load_store (operands, true);
 288     DONE;
 289   }
 290 )
 291
 292 (define_expand "vec_mask_len_store_lanes<mode><vsingle>"
 293   [(match_operand:VT 0 "memory_operand")
 294    (match_operand:VT 1 "register_operand")
 295    (match_operand:<VM> 2 "vector_mask_operand")
 296    (match_operand 3 "autovec_length_operand")
 297    (match_operand 4 "const_0_operand")]
 298   "TARGET_VECTOR"
 299   {
 300     riscv_vector::expand_lanes_load_store (operands, false);
 301     DONE;
 302   }
 303 )
 304
 305 ;; =========================================================================
 306 ;; == Vector creation
 307 ;; =========================================================================
 308
 309 ;; -------------------------------------------------------------------------
 310 ;; ---- [BOOL] Duplicate element
 311 ;; -------------------------------------------------------------------------
 312 ;; The patterns in this section are synthetic.
 313 ;; -------------------------------------------------------------------------
 314
 315 ;; Implement a predicate broadcast by shifting the low bit of the scalar
 316 ;; input into the top bit by duplicate the input and do a compare with zero.
 317 (define_expand "vec_duplicate<mode>"
 318   [(set (match_operand:VB 0 "register_operand")
 319         (vec_duplicate:VB (match_operand:QI 1 "register_operand")))]
 320   "TARGET_VECTOR"
 321   {
 322     poly_int64 nunits = GET_MODE_NUNITS (<MODE>mode);
 323     machine_mode mode = riscv_vector::get_vector_mode (QImode, nunits).require ();
 324     rtx dup = expand_vector_broadcast (mode, operands[1]);
 325     riscv_vector::expand_vec_cmp (operands[0], NE, dup, CONST0_RTX (mode));
 326     DONE;
 327   }
 328 )
 329
 330 ;; -------------------------------------------------------------------------
 331 ;; ---- [INT] Linear series
 332 ;; -------------------------------------------------------------------------
 333 ;; Includes:
 334 ;; - vid.v
 335 ;; - vmul.vx
 336 ;; - vadd.vx/vadd.vi
 337 ;; -------------------------------------------------------------------------
 338
 339 (define_expand "@vec_series<mode>"
 340   [(match_operand:V_VLSI 0 "register_operand")
 341    (match_operand:<VEL> 1 "reg_or_int_operand")
 342    (match_operand:<VEL> 2 "reg_or_int_operand")]
 343   "TARGET_VECTOR"
 344   {
 345     riscv_vector::expand_vec_series (operands[0], operands[1], operands[2]);
 346     DONE;
 347   }
 348 )
 349
 350 ;; -------------------------------------------------------------------------
 351 ;; ---- [INT,FP] permutation
 352 ;; -------------------------------------------------------------------------
 353 ;; This is the pattern permutes the vector
 354 ;; -------------------------------------------------------------------------
 355
 356 (define_expand "vec_perm<mode>"
 357   [(match_operand:V_VLS 0 "register_operand")
 358    (match_operand:V_VLS 1 "register_operand")
 359    (match_operand:V_VLS 2 "register_operand")
 360    (match_operand:<VINDEX> 3 "vector_perm_operand")]
 361   "TARGET_VECTOR && GET_MODE_NUNITS (<MODE>mode).is_constant ()"
 362   {
 363     riscv_vector::expand_vec_perm (operands[0], operands[1],
 364                                    operands[2], operands[3]);
 365     DONE;
 366   }
 367 )
 368
 369 ;; -------------------------------------------------------------------------
 370 ;; ---- [INT,FP] Initialize from individual elements
 371 ;; -------------------------------------------------------------------------
 372 ;; This is the pattern initialize the vector
 373 ;; -------------------------------------------------------------------------
 374
 375 (define_expand "vec_init<mode><vel>"
 376   [(match_operand:V_VLS 0 "register_operand")
 377    (match_operand 1 "")]
 378   "TARGET_VECTOR"
 379   {
 380     riscv_vector::expand_vec_init (operands[0], operands[1]);
 381     DONE;
 382   }
 383 )
 384
 385 ;; Slide an RVV vector left and insert a scalar into element 0.
 386 (define_expand "vec_shl_insert_<mode>"
 387   [(match_operand:VI 0 "register_operand")
 388    (match_operand:VI 1 "register_operand")
 389    (match_operand:<VEL> 2 "reg_or_0_operand")]
 390   "TARGET_VECTOR"
 391 {
 392   insn_code icode = code_for_pred_slide (UNSPEC_VSLIDE1UP, <MODE>mode);
 393   rtx ops[] = {operands[0], operands[1], operands[2]};
 394   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops);
 395   DONE;
 396 })
 397
 398 (define_expand "vec_shl_insert_<mode>"
 399   [(match_operand:VF 0 "register_operand")
 400    (match_operand:VF 1 "register_operand")
 401    (match_operand:<VEL> 2 "register_operand")]
 402   "TARGET_VECTOR"
 403 {
 404   insn_code icode = code_for_pred_slide (UNSPEC_VFSLIDE1UP, <MODE>mode);
 405   rtx ops[] = {operands[0], operands[1], operands[2]};
 406   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops);
 407   DONE;
 408 })
 409
 410 ;; ========================================================================
 411 ;; == Vector operations
 412 ;; =========================================================================
 413
 414 ;; -------------------------------------------------------------------------
 415 ;; ---- [INT] Binary operations
 416 ;; -------------------------------------------------------------------------
 417 ;; Includes:
 418 ;; - vadd.vv/vsub.vv/...
 419 ;; - vadd.vi/vsub.vi/...
 420 ;; -------------------------------------------------------------------------
 421
 422 (define_insn_and_split "<optab><mode>3"
 423   [(set (match_operand:V_VLSI 0 "register_operand")
 424     (any_int_binop_no_shift:V_VLSI
 425      (match_operand:V_VLSI 1 "<binop_rhs1_predicate>")
 426      (match_operand:V_VLSI 2 "<binop_rhs2_predicate>")))]
 427   "TARGET_VECTOR && can_create_pseudo_p ()"
 428   "#"
 429   "&& 1"
 430   [(const_int 0)]
 431 {
 432   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
 433                                  riscv_vector::BINARY_OP, operands);
 434   DONE;
 435 }
 436 [(set_attr "type" "vialu")])
 437
 438 ;; -------------------------------------------------------------------------
 439 ;; ---- [INT] Binary shifts by scalar.
 440 ;; -------------------------------------------------------------------------
 441 ;; Includes:
 442 ;; - vsll.vx/vsra.vx/vsrl.vx
 443 ;; - vsll.vi/vsra.vi/vsrl.vi
 444 ;; -------------------------------------------------------------------------
 445
 446 (define_insn_and_split "<optab><mode>3"
 447   [(set (match_operand:V_VLSI 0 "register_operand"        "=vr")
 448     (any_shift:V_VLSI
 449      (match_operand:V_VLSI 1 "register_operand"           " vr")
 450      (match_operand:<VEL> 2 "vector_scalar_shift_operand" " rK")))]
 451   "TARGET_VECTOR && can_create_pseudo_p ()"
 452   "#"
 453   "&& 1"
 454   [(const_int 0)]
 455 {
 456   operands[2] = gen_lowpart (Pmode, operands[2]);
 457   riscv_vector::emit_vlmax_insn (code_for_pred_scalar (<CODE>, <MODE>mode),
 458                                  riscv_vector::BINARY_OP, operands);
 459   DONE;
 460 }
 461  [(set_attr "type" "vshift")
 462   (set_attr "mode" "<MODE>")])
 463
 464 ;; -------------------------------------------------------------------------
 465 ;; ---- [INT] Binary shifts by vector.
 466 ;; -------------------------------------------------------------------------
 467 ;; Includes:
 468 ;; - vsll.vv/vsra.vv/vsrl.vv
 469 ;; -------------------------------------------------------------------------
 470
 471 (define_insn_and_split "v<optab><mode>3"
 472   [(set (match_operand:V_VLSI 0 "register_operand"  "=vr,vr")
 473     (any_shift:V_VLSI
 474      (match_operand:V_VLSI 1 "register_operand"     " vr,vr")
 475      (match_operand:V_VLSI 2 "vector_shift_operand" " vr,vk")))]
 476   "TARGET_VECTOR && can_create_pseudo_p ()"
 477   "#"
 478   "&& 1"
 479   [(const_int 0)]
 480 {
 481   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
 482                                  riscv_vector::BINARY_OP, operands);
 483   DONE;
 484 }
 485  [(set_attr "type" "vshift")
 486   (set_attr "mode" "<MODE>")])
 487
 488 ;; -------------------------------------------------------------------------
 489 ;; ---- [BOOL] Binary logical operations
 490 ;; -------------------------------------------------------------------------
 491 ;; Includes:
 492 ;; - vmand.mm
 493 ;; - vmxor.mm
 494 ;; - vmor.mm
 495 ;; -------------------------------------------------------------------------
 496
 497 (define_insn_and_split "<optab><mode>3"
 498   [(set (match_operand:VB_VLS 0 "register_operand"                 "=vr")
 499         (any_bitwise:VB_VLS (match_operand:VB_VLS 1 "register_operand" " vr")
 500                             (match_operand:VB_VLS 2 "register_operand" " vr")))]
 501   "TARGET_VECTOR && can_create_pseudo_p ()"
 502   "#"
 503   "&& 1"
 504   [(const_int 0)]
 505   {
 506     insn_code icode = code_for_pred (<CODE>, <MODE>mode);
 507     riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_MASK_OP, operands);
 508     DONE;
 509   }
 510   [(set_attr "type" "vmalu")
 511    (set_attr "mode" "<MODE>")])
 512
 513 ;; -------------------------------------------------------------------------
 514 ;; ---- [BOOL] Inverse
 515 ;; -------------------------------------------------------------------------
 516 ;; Includes:
 517 ;; - vmnot.m
 518 ;; -------------------------------------------------------------------------
 519
 520 (define_insn_and_split "one_cmpl<mode>2"
 521   [(set (match_operand:VB_VLS 0 "register_operand"         "=vr")
 522         (not:VB_VLS (match_operand:VB_VLS 1 "register_operand" " vr")))]
 523   "TARGET_VECTOR && can_create_pseudo_p ()"
 524   "#"
 525   "&& 1"
 526   [(const_int 0)]
 527   {
 528     insn_code icode = code_for_pred_not (<MODE>mode);
 529     riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_MASK_OP, operands);
 530     DONE;
 531   }
 532   [(set_attr "type" "vmalu")
 533    (set_attr "mode" "<MODE>")])
 534
 535 ;; =========================================================================
 536 ;; == Comparisons and selects
 537 ;; =========================================================================
 538
 539 ;; -------------------------------------------------------------------------
 540 ;; ---- [INT,FP] Select based on masks
 541 ;; -------------------------------------------------------------------------
 542 ;; Includes merging patterns for:
 543 ;; - vmerge.vv
 544 ;; - vmerge.vx
 545 ;; - vfmerge.vf
 546 ;; -------------------------------------------------------------------------
 547
 548 (define_insn_and_split "vcond_mask_<mode><vm>"
 549   [(set (match_operand:V_VLS 0 "register_operand")
 550         (if_then_else:V_VLS
 551           (match_operand:<VM> 3 "register_operand")
 552           (match_operand:V_VLS 1 "nonmemory_operand")
 553           (match_operand:V_VLS 2 "register_operand")))]
 554   "TARGET_VECTOR && can_create_pseudo_p ()"
 555   "#"
 556   "&& 1"
 557   [(const_int 0)]
 558   {
 559     /* The order of vcond_mask is opposite to pred_merge.  */
 560     std::swap (operands[1], operands[2]);
 561     riscv_vector::emit_vlmax_insn (code_for_pred_merge (<MODE>mode),
 562                                    riscv_vector::MERGE_OP, operands);
 563     DONE;
 564   }
 565   [(set_attr "type" "vector")]
 566 )
 567
 568 ;; -------------------------------------------------------------------------
 569 ;; ---- [BOOL] Select based on masks
 570 ;; -------------------------------------------------------------------------
 571 ;; Includes merging patterns for:
 572 ;; - vmand.mm
 573 ;; - vmor.mm
 574 ;; - vmnot.m
 575 ;; -------------------------------------------------------------------------
 576
 577 (define_expand "vcond_mask_<mode><mode>"
 578   [(match_operand:VB 0 "register_operand")
 579    (match_operand:VB 1 "register_operand")
 580    (match_operand:VB 2 "register_operand")
 581    (match_operand:VB 3 "register_operand")]
 582   "TARGET_VECTOR"
 583   {
 584     /* mask1 = operands[3] & operands[1].  */
 585     rtx mask1 = expand_binop (<MODE>mode, and_optab, operands[1],
 586                               operands[3], NULL_RTX, 0,
 587                               OPTAB_DIRECT);
 588     /* mask2 = ~operands[3] & operands[2].  */
 589     rtx inverse = expand_unop (<MODE>mode, one_cmpl_optab, operands[3],
 590                                NULL_RTX, 0);
 591     rtx mask2 = expand_binop (<MODE>mode, and_optab, operands[2],
 592                               inverse, NULL_RTX, 0,
 593                               OPTAB_DIRECT);
 594     /* result = mask1 | mask2.  */
 595     rtx result = expand_binop (<MODE>mode, ior_optab, mask1,
 596                                mask2, NULL_RTX, 0,
 597                                OPTAB_DIRECT);
 598     emit_move_insn (operands[0], result);
 599     DONE;
 600   })
 601
 602 ;; -------------------------------------------------------------------------
 603 ;; ---- [INT,FP] Comparisons
 604 ;; -------------------------------------------------------------------------
 605 ;; Includes:
 606 ;; - vms<eq/ne/ltu/lt/leu/le/gtu/gt>.<vv/vx/vi>
 607 ;; -------------------------------------------------------------------------
 608
 609 (define_expand "vec_cmp<mode><vm>"
 610   [(set (match_operand:<VM> 0 "register_operand")
 611         (match_operator:<VM> 1 "comparison_operator"
 612           [(match_operand:V_VLSI 2 "register_operand")
 613            (match_operand:V_VLSI 3 "register_operand")]))]
 614   "TARGET_VECTOR"
 615   {
 616     riscv_vector::expand_vec_cmp (operands[0], GET_CODE (operands[1]),
 617                                   operands[2], operands[3]);
 618     DONE;
 619   }
 620 )
 621
 622 (define_expand "vec_cmpu<mode><vm>"
 623   [(set (match_operand:<VM> 0 "register_operand")
 624         (match_operator:<VM> 1 "comparison_operator"
 625           [(match_operand:V_VLSI 2 "register_operand")
 626            (match_operand:V_VLSI 3 "register_operand")]))]
 627   "TARGET_VECTOR"
 628   {
 629     riscv_vector::expand_vec_cmp (operands[0], GET_CODE (operands[1]),
 630                                   operands[2], operands[3]);
 631     DONE;
 632   }
 633 )
 634
 635 (define_expand "vec_cmp<mode><vm>"
 636   [(set (match_operand:<VM> 0 "register_operand")
 637         (match_operator:<VM> 1 "comparison_operator"
 638           [(match_operand:V_VLSF 2 "register_operand")
 639            (match_operand:V_VLSF 3 "register_operand")]))]
 640   "TARGET_VECTOR"
 641   {
 642     riscv_vector::expand_vec_cmp_float (operands[0], GET_CODE (operands[1]),
 643                                         operands[2], operands[3], false);
 644     DONE;
 645   }
 646 )
 647
 648 ;; -------------------------------------------------------------------------
 649 ;; ---- [INT] Sign and zero extension
 650 ;; -------------------------------------------------------------------------
 651 ;; Includes:
 652 ;; - vzext.vf[2|4|8]
 653 ;; - vsext.vf[2|4|8]
 654 ;; -------------------------------------------------------------------------
 655
 656 ;; Use define_insn_and_split to define vsext.vf2/vzext.vf2 will help
 657 ;; to combine instructions as below:
 658 ;;   vsext.vf2 + vsext.vf2 + vadd.vv ==> vwadd.vv
 659 (define_insn_and_split "<optab><v_double_trunc><mode>2"
 660   [(set (match_operand:VWEXTI 0 "register_operand" "=&vr")
 661     (any_extend:VWEXTI
 662      (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand" "vr")))]
 663   "TARGET_VECTOR && can_create_pseudo_p ()"
 664   "#"
 665   "&& 1"
 666   [(const_int 0)]
 667 {
 668   insn_code icode = code_for_pred_vf2 (<CODE>, <MODE>mode);
 669   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 670   DONE;
 671 }
 672   [(set_attr "type" "vext")
 673    (set_attr "mode" "<MODE>")])
 674
 675 (define_insn_and_split "<optab><v_quad_trunc><mode>2"
 676   [(set (match_operand:VQEXTI 0 "register_operand")
 677     (any_extend:VQEXTI
 678      (match_operand:<V_QUAD_TRUNC> 1 "register_operand")))]
 679   "TARGET_VECTOR && can_create_pseudo_p ()"
 680   "#"
 681   "&& 1"
 682   [(const_int 0)]
 683 {
 684   insn_code icode = code_for_pred_vf4 (<CODE>, <MODE>mode);
 685   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 686   DONE;
 687 }
 688 [(set_attr "type" "vext")])
 689
 690 (define_insn_and_split "<optab><v_oct_trunc><mode>2"
 691   [(set (match_operand:VOEXTI 0 "register_operand")
 692     (any_extend:VOEXTI
 693      (match_operand:<V_OCT_TRUNC> 1 "register_operand")))]
 694   "TARGET_VECTOR && can_create_pseudo_p ()"
 695   "#"
 696   "&& 1"
 697   [(const_int 0)]
 698 {
 699   insn_code icode = code_for_pred_vf8 (<CODE>, <MODE>mode);
 700   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 701   DONE;
 702 }
 703 [(set_attr "type" "vext")])
 704
 705 ;; -------------------------------------------------------------------------
 706 ;; ---- [INT] Truncation
 707 ;; -------------------------------------------------------------------------
 708 ;; - vncvt.x.x.w
 709 ;; -------------------------------------------------------------------------
 710 (define_insn_and_split "trunc<mode><v_double_trunc>2"
 711   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand" "=vr")
 712     (truncate:<V_DOUBLE_TRUNC>
 713      (match_operand:VWEXTI 1 "register_operand"              " vr")))]
 714   "TARGET_VECTOR && can_create_pseudo_p ()"
 715   "#"
 716   "&& 1"
 717   [(const_int 0)]
 718 {
 719   insn_code icode = code_for_pred_trunc (<MODE>mode);
 720   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 721   DONE;
 722 }
 723   [(set_attr "type" "vshift")
 724    (set_attr "mode" "<MODE>")])
 725
 726 ;; -------------------------------------------------------------------------
 727 ;; Truncation to a mode whose inner mode size is a quarter of mode's.
 728 ;; We emulate this with two consecutive vncvts.
 729 ;; -------------------------------------------------------------------------
 730 (define_expand "trunc<mode><v_quad_trunc>2"
 731   [(set (match_operand:<V_QUAD_TRUNC> 0 "register_operand")
 732     (truncate:<V_QUAD_TRUNC>
 733      (match_operand:VQEXTI 1 "register_operand")))]
 734   "TARGET_VECTOR"
 735 {
 736   rtx half = gen_reg_rtx (<V_DOUBLE_TRUNC>mode);
 737   emit_insn (gen_trunc<mode><v_double_trunc>2 (half, operands[1]));
 738   emit_insn (gen_trunc<v_double_trunc><v_quad_trunc>2 (operands[0], half));
 739   DONE;
 740 })
 741
 742 ;; -------------------------------------------------------------------------
 743 ;; Truncation to a mode whose inner mode size is an eigth of mode's.
 744 ;; We emulate this with three consecutive vncvts.
 745 ;; -------------------------------------------------------------------------
 746 (define_expand "trunc<mode><v_oct_trunc>2"
 747   [(set (match_operand:<V_OCT_TRUNC> 0 "register_operand")
 748     (truncate:<V_OCT_TRUNC>
 749      (match_operand:VOEXTI 1 "register_operand")))]
 750   "TARGET_VECTOR"
 751 {
 752   rtx quarter = gen_reg_rtx (<V_QUAD_TRUNC>mode);
 753   emit_insn (gen_trunc<mode><v_quad_trunc>2 (quarter, operands[1]));
 754   emit_insn (gen_trunc<v_quad_trunc><v_oct_trunc>2 (operands[0], quarter));
 755   DONE;
 756 })
 757
 758 ;; -------------------------------------------------------------------------
 759 ;; ---- [FP] Widening.
 760 ;; -------------------------------------------------------------------------
 761 ;; - vfwcvt.f.f.v
 762 ;; -------------------------------------------------------------------------
 763 (define_insn_and_split "extend<v_double_trunc><mode>2"
 764   [(set (match_operand:VWEXTF_ZVFHMIN 0 "register_operand" "=&vr")
 765     (float_extend:VWEXTF_ZVFHMIN
 766      (match_operand:<V_DOUBLE_TRUNC>  1 "register_operand" "  vr")))]
 767   "TARGET_VECTOR && can_create_pseudo_p ()"
 768   "#"
 769   "&& 1"
 770   [(const_int 0)]
 771 {
 772   insn_code icode = code_for_pred_extend (<MODE>mode);
 773   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 774   DONE;
 775 }
 776   [(set_attr "type" "vfwcvtftof")
 777    (set_attr "mode" "<MODE>")])
 778
 779 (define_expand "extend<v_quad_trunc><mode>2"
 780   [(set (match_operand:VQEXTF 0 "register_operand")
 781     (float_extend:VQEXTF
 782      (match_operand:<V_QUAD_TRUNC> 1 "register_operand")))]
 783   "TARGET_VECTOR && (TARGET_ZVFHMIN || TARGET_ZVFH)"
 784 {
 785   rtx dblw = gen_reg_rtx (<V_DOUBLE_TRUNC>mode);
 786   emit_insn (gen_extend<v_quad_trunc><v_double_trunc>2 (dblw, operands[1]));
 787   emit_insn (gen_extend<v_double_trunc><mode>2 (operands[0], dblw));
 788   DONE;
 789 })
 790
 791 ;; -------------------------------------------------------------------------
 792 ;; ---- [FP] Narrowing.
 793 ;; -------------------------------------------------------------------------
 794 ;; - vfncvt.f.f.w
 795 ;; -------------------------------------------------------------------------
 796 (define_insn_and_split "trunc<mode><v_double_trunc>2"
 797   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand" "=vr")
 798     (float_truncate:<V_DOUBLE_TRUNC>
 799      (match_operand:VWEXTF_ZVFHMIN 1 "register_operand"      " vr")))]
 800   "TARGET_VECTOR && can_create_pseudo_p ()"
 801   "#"
 802   "&& 1"
 803   [(const_int 0)]
 804 {
 805   insn_code icode = code_for_pred_trunc (<MODE>mode);
 806   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
 807   DONE;
 808 }
 809   [(set_attr "type" "vfncvtftof")
 810    (set_attr "mode" "<MODE>")])
 811
 812 ;; -------------------------------------------------------------------------
 813 ;; Narrowing to a mode whose inner mode size is a quarter of mode's.
 814 ;; We emulate this with two consecutive vfncvts.
 815 ;; -------------------------------------------------------------------------
 816 (define_expand "trunc<mode><v_quad_trunc>2"
 817   [(set (match_operand:<V_QUAD_TRUNC> 0 "register_operand")
 818     (float_truncate:<V_QUAD_TRUNC>
 819      (match_operand:VQEXTF 1 "register_operand")))]
 820   "TARGET_VECTOR && (TARGET_ZVFHMIN || TARGET_ZVFH)"
 821 {
 822   rtx half = gen_reg_rtx (<V_DOUBLE_TRUNC>mode);
 823   rtx opshalf[] = {half, operands[1]};
 824
 825   /* According to the RISC-V V Spec 13.19. we need to use
 826      vfncvt.rod.f.f.w for all steps but the last.  */
 827   insn_code icode = code_for_pred_rod_trunc (<MODE>mode);
 828   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, opshalf);
 829
 830   emit_insn (gen_trunc<v_double_trunc><v_quad_trunc>2 (operands[0], half));
 831   DONE;
 832 })
 833
 834
 835 ;; =========================================================================
 836 ;; == Conversions
 837 ;; =========================================================================
 838
 839 ;; -------------------------------------------------------------------------
 840 ;; ---- [INT<-FP] Conversions
 841 ;; -------------------------------------------------------------------------
 842 ;; Includes:
 843 ;; - vfcvt.rtz.xu.f.v
 844 ;; - vfcvt.rtz.x.f.v
 845 ;; -------------------------------------------------------------------------
 846
 847 (define_insn_and_split "<optab><mode><vconvert>2"
 848   [(set (match_operand:<VCONVERT> 0 "register_operand")
 849         (any_fix:<VCONVERT>
 850           (match_operand:VF 1 "register_operand")))]
 851   "TARGET_VECTOR && can_create_pseudo_p ()"
 852   "#"
 853   "&& 1"
 854   [(const_int 0)]
 855 {
 856   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
 857   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 858   DONE;
 859 }
 860 [(set_attr "type" "vfcvtftoi")])
 861
 862 ;; -------------------------------------------------------------------------
 863 ;; ---- [FP<-INT] Conversions
 864 ;; -------------------------------------------------------------------------
 865 ;; Includes:
 866 ;; - vfcvt.f.xu.v
 867 ;; - vfcvt.f.x.v
 868 ;; -------------------------------------------------------------------------
 869
 870 (define_insn_and_split "<float_cvt><vconvert><mode>2"
 871   [(set (match_operand:VF 0 "register_operand")
 872         (any_float:VF
 873           (match_operand:<VCONVERT> 1 "register_operand")))]
 874   "TARGET_VECTOR && can_create_pseudo_p ()"
 875   "#"
 876   "&& 1"
 877   [(const_int 0)]
 878 {
 879   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
 880   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
 881   DONE;
 882 }
 883 [(set_attr "type" "vfcvtitof")])
 884
 885 ;; =========================================================================
 886 ;; == Widening/narrowing Conversions
 887 ;; =========================================================================
 888
 889 ;; -------------------------------------------------------------------------
 890 ;; ---- [INT<-FP] Widening Conversions
 891 ;; -------------------------------------------------------------------------
 892 ;; Includes:
 893 ;; - vfwcvt.rtz.xu.f.v
 894 ;; - vfwcvt.rtz.x.f.v
 895 ;; -------------------------------------------------------------------------
 896 (define_insn_and_split "<optab><vnconvert><mode>2"
 897   [(set (match_operand:VWCONVERTI 0 "register_operand")
 898         (any_fix:VWCONVERTI
 899           (match_operand:<VNCONVERT> 1 "register_operand")))]
 900   "TARGET_VECTOR && can_create_pseudo_p ()"
 901   "#"
 902   "&& 1"
 903   [(const_int 0)]
 904 {
 905   insn_code icode = code_for_pred_widen (<CODE>, <MODE>mode);
 906   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 907   DONE;
 908 }
 909 [(set_attr "type" "vfwcvtftoi")])
 910
 911 ;; -------------------------------------------------------------------------
 912 ;; ---- [FP<-INT] Widening Conversions
 913 ;; -------------------------------------------------------------------------
 914 ;; Includes:
 915 ;; - vfwcvt.f.xu.v
 916 ;; - vfwcvt.f.x.v
 917 ;; -------------------------------------------------------------------------
 918 (define_insn_and_split "<float_cvt><vnconvert><mode>2"
 919   [(set (match_operand:VF 0 "register_operand")
 920         (any_float:VF
 921           (match_operand:<VNCONVERT> 1 "register_operand")))]
 922   "TARGET_VECTOR && can_create_pseudo_p ()"
 923   "#"
 924   "&& 1"
 925   [(const_int 0)]
 926 {
 927   insn_code icode = code_for_pred_widen (<CODE>, <MODE>mode);
 928   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 929   DONE;
 930 }
 931 [(set_attr "type" "vfwcvtitof")])
 932
 933 ;; -------------------------------------------------------------------------
 934 ;; ---- [INT<-FP] Narrowing Conversions
 935 ;; -------------------------------------------------------------------------
 936 ;; Includes:
 937 ;; - vfncvt.rtz.xu.f.v
 938 ;; - vfncvt.rtz.x.f.v
 939 ;; -------------------------------------------------------------------------
 940 (define_insn_and_split "<optab><mode><vnconvert>2"
 941   [(set (match_operand:<VNCONVERT> 0 "register_operand")
 942         (any_fix:<VNCONVERT>
 943           (match_operand:VF 1 "register_operand")))]
 944   "TARGET_VECTOR && can_create_pseudo_p ()"
 945   "#"
 946   "&& 1"
 947   [(const_int 0)]
 948 {
 949   insn_code icode = code_for_pred_narrow (<CODE>, <MODE>mode);
 950   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 951   DONE;
 952 }
 953 [(set_attr "type" "vfncvtftoi")])
 954
 955 ;; -------------------------------------------------------------------------
 956 ;; ---- [FP<-INT] Narrowing Conversions
 957 ;; -------------------------------------------------------------------------
 958 ;; Includes:
 959 ;; - vfncvt.f.xu.w
 960 ;; - vfncvt.f.x.w
 961 ;; -------------------------------------------------------------------------
 962 (define_insn_and_split "<float_cvt><mode><vnconvert>2"
 963   [(set (match_operand:<VNCONVERT> 0 "register_operand")
 964         (any_float:<VNCONVERT>
 965           (match_operand:VWCONVERTI 1 "register_operand")))]
 966   "TARGET_VECTOR && can_create_pseudo_p ()"
 967   "#"
 968   "&& 1"
 969   [(const_int 0)]
 970 {
 971   insn_code icode = code_for_pred_narrow (<CODE>, <MODE>mode);
 972   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
 973   DONE;
 974 }
 975 [(set_attr "type" "vfncvtitof")])
 976
 977 ;; =========================================================================
 978 ;; == Unary arithmetic
 979 ;; =========================================================================
 980
 981 ;; -------------------------------------------------------------------------------
 982 ;; ---- [INT] Unary operations
 983 ;; -------------------------------------------------------------------------------
 984 ;; Includes:
 985 ;; - vneg.v/vnot.v
 986 ;; -------------------------------------------------------------------------------
 987 (define_insn_and_split "<optab><mode>2"
 988   [(set (match_operand:V_VLSI 0 "register_operand")
 989     (any_int_unop:V_VLSI
 990      (match_operand:V_VLSI 1 "register_operand")))]
 991   "TARGET_VECTOR && can_create_pseudo_p ()"
 992   "#"
 993   "&& 1"
 994   [(const_int 0)]
 995 {
 996   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
 997   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
 998   DONE;
 999 }
1000 [(set_attr "type" "vialu")])
1001
1002 ;; -------------------------------------------------------------------------------
1003 ;; - [INT] ABS expansion to vmslt and vneg.
1004 ;; -------------------------------------------------------------------------------
1005
1006 (define_insn_and_split "abs<mode>2"
1007   [(set (match_operand:VI 0 "register_operand")
1008      (abs:VI
1009        (match_operand:VI 1 "register_operand")))]
1010   "TARGET_VECTOR && can_create_pseudo_p ()"
1011   "#"
1012   "&& 1"
1013   [(const_int 0)]
1014 {
1015   rtx zero = gen_const_vec_duplicate (<MODE>mode, GEN_INT (0));
1016   machine_mode mask_mode = riscv_vector::get_mask_mode (<MODE>mode);
1017   rtx mask = gen_reg_rtx (mask_mode);
1018   riscv_vector::expand_vec_cmp (mask, LT, operands[1], zero);
1019
1020   rtx ops[] = {operands[0], mask, operands[1], operands[1]};
1021   riscv_vector::emit_vlmax_insn (code_for_pred (NEG, <MODE>mode),
1022                                   riscv_vector::UNARY_OP_TAMU, ops);
1023   DONE;
1024 }
1025 [(set_attr "type" "vector")])
1026
1027 ;; -------------------------------------------------------------------------------
1028 ;; ---- [FP] Unary operations
1029 ;; -------------------------------------------------------------------------------
1030 ;; Includes:
1031 ;; - vfneg.v/vfabs.v
1032 ;; -------------------------------------------------------------------------------
1033 (define_insn_and_split "<optab><mode>2"
1034   [(set (match_operand:VF 0 "register_operand")
1035     (any_float_unop_nofrm:VF
1036      (match_operand:VF 1 "register_operand")))]
1037   "TARGET_VECTOR && can_create_pseudo_p ()"
1038   "#"
1039   "&& 1"
1040   [(const_int 0)]
1041 {
1042   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1043   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, operands);
1044   DONE;
1045 }
1046 [(set_attr "type" "vector")])
1047
1048 ;; -------------------------------------------------------------------------------
1049 ;; - [FP] Square root
1050 ;; -------------------------------------------------------------------------------
1051 ;; Includes:
1052 ;; - vfsqrt.v
1053 ;; -------------------------------------------------------------------------------
1054 (define_insn_and_split "<optab><mode>2"
1055   [(set (match_operand:VF 0 "register_operand")
1056     (any_float_unop:VF
1057      (match_operand:VF 1 "register_operand")))]
1058   "TARGET_VECTOR && can_create_pseudo_p ()"
1059   "#"
1060   "&& 1"
1061   [(const_int 0)]
1062 {
1063   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1064   riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP_FRM_DYN, operands);
1065   DONE;
1066 }
1067 [(set_attr "type" "vfsqrt")])
1068
1069 ;; =========================================================================
1070 ;; == Ternary arithmetic
1071 ;; =========================================================================
1072
1073 ;; -------------------------------------------------------------------------
1074 ;; ---- [INT] VMACC and VMADD
1075 ;; -------------------------------------------------------------------------
1076 ;; Includes:
1077 ;; - vmacc
1078 ;; - vmadd
1079 ;; -------------------------------------------------------------------------
1080
1081 (define_insn_and_split "fma<mode>4"
1082   [(set (match_operand:V_VLSI 0 "register_operand")
1083         (plus:V_VLSI
1084           (mult:V_VLSI
1085             (match_operand:V_VLSI 1 "register_operand")
1086             (match_operand:V_VLSI 2 "register_operand"))
1087           (match_operand:V_VLSI 3 "register_operand")))]
1088   "TARGET_VECTOR && can_create_pseudo_p ()"
1089   "#"
1090   "&& 1"
1091   [(const_int 0)]
1092   {
1093     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1094                  RVV_VUNDEF(<MODE>mode)};
1095     riscv_vector::emit_vlmax_insn (code_for_pred_mul_plus (<MODE>mode),
1096                                    riscv_vector::TERNARY_OP, ops);
1097     DONE;
1098   }
1099   [(set_attr "type" "vector")])
1100
1101 ;; -------------------------------------------------------------------------
1102 ;; ---- [INT] VNMSAC and VNMSUB
1103 ;; -------------------------------------------------------------------------
1104 ;; Includes:
1105 ;; - vnmsac
1106 ;; - vnmsub
1107 ;; -------------------------------------------------------------------------
1108
1109 (define_insn_and_split "fnma<mode>4"
1110   [(set (match_operand:V_VLSI 0 "register_operand")
1111         (minus:V_VLSI
1112           (match_operand:V_VLSI 3 "register_operand")
1113           (mult:V_VLSI
1114             (match_operand:V_VLSI 1 "register_operand")
1115             (match_operand:V_VLSI 2 "register_operand"))))]
1116   "TARGET_VECTOR && can_create_pseudo_p ()"
1117   "#"
1118   "&& 1"
1119   [(const_int 0)]
1120   {
1121     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1122                  RVV_VUNDEF(<MODE>mode)};
1123     riscv_vector::emit_vlmax_insn (code_for_pred_minus_mul (<MODE>mode),
1124                                    riscv_vector::TERNARY_OP, ops);
1125     DONE;
1126   }
1127   [(set_attr "type" "vector")])
1128
1129 ;; -------------------------------------------------------------------------
1130 ;; ---- [FP] VFMACC and VFMADD
1131 ;; -------------------------------------------------------------------------
1132 ;; Includes:
1133 ;; - vfmacc
1134 ;; - vfmadd
1135 ;; -------------------------------------------------------------------------
1136
1137 (define_insn_and_split "fma<mode>4"
1138   [(set (match_operand:VF 0 "register_operand")
1139         (plus:VF
1140           (mult:VF
1141             (match_operand:VF 1 "register_operand")
1142             (match_operand:VF 2 "register_operand"))
1143           (match_operand:VF 3 "register_operand")))]
1144   "TARGET_VECTOR && can_create_pseudo_p ()"
1145   "#"
1146   "&& 1"
1147   [(const_int 0)]
1148   {
1149     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1150                  RVV_VUNDEF(<MODE>mode)};
1151     riscv_vector::emit_vlmax_insn (code_for_pred_mul (PLUS, <MODE>mode),
1152                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1153     DONE;
1154   }
1155   [(set_attr "type" "vector")])
1156
1157 ;; -------------------------------------------------------------------------
1158 ;; ---- [FP] VFNMSAC and VFNMSUB
1159 ;; -------------------------------------------------------------------------
1160 ;; Includes:
1161 ;; - vfnmsac
1162 ;; - vfnmsub
1163 ;; -------------------------------------------------------------------------
1164
1165 (define_insn_and_split "fnma<mode>4"
1166   [(set (match_operand:VF 0 "register_operand")
1167         (minus:VF
1168           (match_operand:VF 3 "register_operand")
1169           (mult:VF
1170             (match_operand:VF 1 "register_operand")
1171             (match_operand:VF 2 "register_operand"))))]
1172   "TARGET_VECTOR && can_create_pseudo_p ()"
1173   "#"
1174   "&& 1"
1175   [(const_int 0)]
1176   {
1177     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1178                  RVV_VUNDEF(<MODE>mode)};
1179     riscv_vector::emit_vlmax_insn (code_for_pred_mul_neg (PLUS, <MODE>mode),
1180                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1181     DONE;
1182   }
1183   [(set_attr "type" "vector")])
1184
1185 ;; -------------------------------------------------------------------------
1186 ;; ---- [FP] VFMSAC and VFMSUB
1187 ;; -------------------------------------------------------------------------
1188 ;; Includes:
1189 ;; - vfmsac
1190 ;; - vfmsub
1191 ;; -------------------------------------------------------------------------
1192
1193 (define_insn_and_split "fms<mode>4"
1194   [(set (match_operand:VF 0 "register_operand")
1195         (minus:VF
1196           (mult:VF
1197             (match_operand:VF 1 "register_operand")
1198             (match_operand:VF 2 "register_operand"))
1199           (match_operand:VF 3 "register_operand")))]
1200   "TARGET_VECTOR && can_create_pseudo_p ()"
1201   "#"
1202   "&& 1"
1203   [(const_int 0)]
1204   {
1205     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1206                  RVV_VUNDEF(<MODE>mode)};
1207     riscv_vector::emit_vlmax_insn (code_for_pred_mul (MINUS, <MODE>mode),
1208                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1209     DONE;
1210   }
1211   [(set_attr "type" "vector")])
1212
1213 ;; -------------------------------------------------------------------------
1214 ;; ---- [FP] VFNMACC and VFNMADD
1215 ;; -------------------------------------------------------------------------
1216 ;; Includes:
1217 ;; - vfnmacc
1218 ;; - vfnmadd
1219 ;; -------------------------------------------------------------------------
1220
1221 (define_insn_and_split "fnms<mode>4"
1222   [(set (match_operand:VF 0 "register_operand")
1223         (minus:VF
1224           (neg:VF
1225             (mult:VF
1226               (match_operand:VF 1 "register_operand")
1227               (match_operand:VF 2 "register_operand")))
1228           (match_operand:VF 3 "register_operand")))]
1229   "TARGET_VECTOR && can_create_pseudo_p ()"
1230   "#"
1231   "&& 1"
1232   [(const_int 0)]
1233   {
1234     rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
1235                  RVV_VUNDEF(<MODE>mode)};
1236     riscv_vector::emit_vlmax_insn (code_for_pred_mul_neg (MINUS, <MODE>mode),
1237                                    riscv_vector::TERNARY_OP_FRM_DYN, ops);
1238     DONE;
1239   }
1240   [(set_attr "type" "vector")])
1241
1242 ;; =========================================================================
1243 ;; == SELECT_VL
1244 ;; =========================================================================
1245
1246 (define_expand "select_vl<mode>"
1247   [(match_operand:P 0 "register_operand")
1248    (match_operand:P 1 "vector_length_operand")
1249    (match_operand:P 2 "")]
1250   "TARGET_VECTOR"
1251 {
1252   riscv_vector::expand_select_vl (operands);
1253   DONE;
1254 })
1255
1256 ;; -------------------------------------------------------------------------
1257 ;; ---- [INT,FP] Insert a vector element.
1258 ;; -------------------------------------------------------------------------
1259
1260 (define_expand "vec_set<mode>"
1261   [(match_operand:V_VLS 0 "register_operand")
1262    (match_operand:<VEL> 1 "register_operand")
1263    (match_operand       2 "nonmemory_operand")]
1264   "TARGET_VECTOR"
1265 {
1266   /* If we set the first element, emit an v(f)mv.s.[xf].  */
1267   if (operands[2] == const0_rtx)
1268     {
1269       rtx ops[] = {operands[0], operands[0], operands[1]};
1270       riscv_vector::emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
1271                                         riscv_vector::SCALAR_MOVE_MERGED_OP, ops, CONST1_RTX (Pmode));
1272     }
1273   else
1274     {
1275       /* Move the desired value into a vector register and insert
1276          it at the proper position using vslideup with an
1277          "effective length" of 1 i.e. a VL 1 past the offset.  */
1278
1279       /* Here we set VL = offset + 1.  */
1280       rtx length = gen_reg_rtx (Pmode);
1281       operands[2] = gen_lowpart (Pmode, operands[2]);
1282       if (CONST_INT_P (operands[2]))
1283           emit_move_insn (length, GEN_INT (INTVAL (operands[2]) + 1));
1284       else
1285         {
1286           rtx add = gen_rtx_PLUS (GET_MODE (operands[2]),
1287                                   operands[2], GEN_INT (1));
1288           emit_move_insn (length, add);
1289         }
1290
1291       /* Move operands[1] into a vector register via vmv.v.x using the same
1292          VL we need for the slide.  */
1293       rtx tmp = gen_reg_rtx (<MODE>mode);
1294       rtx ops1[] = {tmp, operands[1]};
1295       emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
1296                            riscv_vector::UNARY_OP, ops1, length);
1297
1298       /* Slide exactly one element up leaving the tail elements
1299          unchanged.  */
1300       rtx ops2[] = {operands[0], operands[0], tmp, operands[2]};
1301       riscv_vector::emit_nonvlmax_insn
1302         (code_for_pred_slide (UNSPEC_VSLIDEUP, <MODE>mode), riscv_vector::BINARY_OP_TUMA, ops2, length);
1303     }
1304   DONE;
1305 })
1306
1307 ;; -------------------------------------------------------------------------
1308 ;; ---- [INT,FP] Extract a vector element.
1309 ;; -------------------------------------------------------------------------
1310 (define_expand "vec_extract<mode><vel>"
1311   [(set (match_operand:<VEL>      0 "register_operand")
1312      (vec_select:<VEL>
1313        (match_operand:V_VLS       1 "register_operand")
1314        (parallel
1315          [(match_operand          2 "nonmemory_operand")])))]
1316   "TARGET_VECTOR"
1317 {
1318   /* Element extraction can be done by sliding down the requested element
1319      to index 0 and then v(f)mv.[xf].s it to a scalar register.  */
1320
1321   /* When extracting any other than the first element we need to slide
1322      it down.  */
1323   rtx tmp = NULL_RTX;
1324   if (operands[2] != const0_rtx)
1325     {
1326       /* Emit the slide down to index 0 in a new vector.  */
1327       tmp = gen_reg_rtx (<MODE>mode);
1328       operands[2] = gen_lowpart (Pmode, operands[2]);
1329       rtx ops[] = {tmp, operands[1], operands[2]};
1330       riscv_vector::emit_vlmax_insn
1331         (code_for_pred_slide (UNSPEC_VSLIDEDOWN, <MODE>mode), riscv_vector::BINARY_OP, ops);
1332     }
1333
1334   /* Emit v(f)mv.[xf].s.  */
1335   emit_insn (gen_pred_extract_first (<MODE>mode, operands[0],
1336                                      tmp ? tmp : operands[1]));
1337   DONE;
1338 })
1339
1340 ;; -------------------------------------------------------------------------
1341 ;; This extracts a bit (via QImode) from a bitmask vector.
1342 ;; -------------------------------------------------------------------------
1343 (define_expand "vec_extract<mode>qi"
1344   [(set (match_operand:QI         0 "register_operand")
1345      (vec_select:QI
1346        (match_operand:VB          1 "register_operand")
1347        (parallel
1348          [(match_operand          2 "nonmemory_operand")])))]
1349   "TARGET_VECTOR"
1350 {
1351   /* Create an empty byte vector and set it to one under mask.  */
1352   machine_mode qimode = riscv_vector::get_vector_mode
1353       (QImode, GET_MODE_NUNITS (<MODE>mode)).require ();
1354
1355   rtx tmp1 = gen_reg_rtx (qimode);
1356   emit_move_insn (tmp1, gen_const_vec_duplicate (qimode, GEN_INT (0)));
1357   rtx ones = gen_const_vec_duplicate (qimode, GEN_INT (1));
1358
1359   rtx ops1[] = {tmp1, tmp1, ones, operands[1]};
1360   riscv_vector::emit_vlmax_insn (code_for_pred_merge (qimode),
1361                                  riscv_vector::MERGE_OP, ops1);
1362
1363   /* Slide down the requested byte element.  */
1364   rtx tmp2 = gen_reg_rtx (qimode);
1365
1366   rtx ops2[] = {tmp2, tmp1, operands[2]};
1367   riscv_vector::emit_vlmax_insn
1368     (code_for_pred_slide (UNSPEC_VSLIDEDOWN, qimode),
1369      riscv_vector::BINARY_OP, ops2);
1370
1371   /* Extract it.  */
1372   emit_insn (gen_pred_extract_first (qimode, operands[0], tmp2));
1373   DONE;
1374 })
1375
1376 ;; -------------------------------------------------------------------------
1377 ;; ---- [FP] Binary operations
1378 ;; -------------------------------------------------------------------------
1379 ;; Includes:
1380 ;; - vfadd.vv/vfsub.vv/...
1381 ;; - vfadd.vf/vfsub.vf/...
1382 ;; -------------------------------------------------------------------------
1383 (define_insn_and_split "<optab><mode>3"
1384   [(set (match_operand:V_VLSF 0 "register_operand")
1385         (any_float_binop:V_VLSF
1386           (match_operand:V_VLSF 1 "register_operand")
1387           (match_operand:V_VLSF 2 "register_operand")))]
1388   "TARGET_VECTOR && can_create_pseudo_p ()"
1389   "#"
1390   "&& 1"
1391   [(const_int 0)]
1392 {
1393   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
1394                                     riscv_vector::BINARY_OP_FRM_DYN, operands);
1395   DONE;
1396 }
1397 [(set_attr "type" "vfalu")])
1398
1399 ;; -------------------------------------------------------------------------
1400 ;; Includes:
1401 ;; - vfmin.vv/vfmax.vv
1402 ;; - vfmin.vf/vfmax.vf
1403 ;; -------------------------------------------------------------------------
1404 (define_insn_and_split "<optab><mode>3"
1405   [(set (match_operand:V_VLSF 0 "register_operand")
1406         (any_float_binop_nofrm:V_VLSF
1407           (match_operand:V_VLSF 1 "register_operand")
1408           (match_operand:V_VLSF 2 "register_operand")))]
1409   "TARGET_VECTOR && can_create_pseudo_p ()"
1410   "#"
1411   "&& 1"
1412   [(const_int 0)]
1413 {
1414   riscv_vector::emit_vlmax_insn (code_for_pred (<CODE>, <MODE>mode),
1415                                   riscv_vector::BINARY_OP, operands);
1416   DONE;
1417 }
1418 [(set_attr "type" "vfminmax")])
1419
1420 ;; -------------------------------------------------------------------------------
1421 ;; ---- [FP] Sign copying
1422 ;; -------------------------------------------------------------------------------
1423 ;; Includes:
1424 ;; - vfsgnj.vv/vfsgnjn.vv
1425 ;; - vfsgnj.vf/vfsgnjn.vf
1426 ;; -------------------------------------------------------------------------------
1427
1428 ;; Leave the pattern like this as to still allow combine to match
1429 ;; a negated copysign (see vector.md) before adding the UNSPEC_VPREDICATE later.
1430 (define_insn_and_split "copysign<mode>3"
1431   [(set (match_operand:V_VLSF 0 "register_operand"  "=vd, vd, vr, vr")
1432     (unspec:V_VLSF
1433      [(match_operand:V_VLSF 1 "register_operand"        " vr, vr, vr, vr")
1434      (match_operand:V_VLSF 2 "register_operand"         " vr, vr, vr, vr")] UNSPEC_VCOPYSIGN))]
1435   "TARGET_VECTOR && can_create_pseudo_p ()"
1436   "#"
1437   "&& 1"
1438   [(const_int 0)]
1439 {
1440   riscv_vector::emit_vlmax_insn (code_for_pred (UNSPEC_VCOPYSIGN, <MODE>mode),
1441                                   riscv_vector::BINARY_OP, operands);
1442   DONE;
1443 }
1444   [(set_attr "type" "vfsgnj")
1445    (set_attr "mode" "<MODE>")])
1446
1447 ;; -------------------------------------------------------------------------------
1448 ;; Includes:
1449 ;; - vfsgnjx.vv
1450 ;; - vfsgnjx.vf
1451 ;; -------------------------------------------------------------------------------
1452 (define_expand "xorsign<mode>3"
1453   [(match_operand:V_VLSF 0 "register_operand")
1454     (match_operand:V_VLSF 1 "register_operand")
1455     (match_operand:V_VLSF 2 "register_operand")]
1456   "TARGET_VECTOR"
1457 {
1458   riscv_vector::emit_vlmax_insn (code_for_pred (UNSPEC_VXORSIGN, <MODE>mode),
1459                                   riscv_vector::BINARY_OP, operands);
1460   DONE;
1461 })
1462
1463 ;; -------------------------------------------------------------------------
1464 ;; ---- [INT] Highpart multiplication
1465 ;; -------------------------------------------------------------------------
1466 ;; Includes:
1467 ;; - vmulh.vv
1468 ;; - vmulhu.vv
1469 ;; -------------------------------------------------------------------------
1470
1471 (define_insn_and_split "<mulh_table><mode>3_highpart"
1472   [(set (match_operand:VFULLI 0 "register_operand")
1473         (mulh:VFULLI
1474           (match_operand:VFULLI 1 "register_operand")
1475           (match_operand:VFULLI 2 "register_operand")))]
1476   "TARGET_VECTOR && can_create_pseudo_p ()"
1477   "#"
1478   "&& 1"
1479   [(const_int 0)]
1480 {
1481   insn_code icode = code_for_pred_mulh (<MULH_UNSPEC>, <MODE>mode);
1482   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
1483   DONE;
1484 }
1485 [(set_attr "type" "vimul")])
1486
1487 ;; -------------------------------------------------------------------------
1488 ;; ---- [INT] Conditional unary operations
1489 ;; -------------------------------------------------------------------------
1490 ;; Includes:
1491 ;; - vneg/vnot
1492 ;; -------------------------------------------------------------------------
1493
1494 (define_expand "cond_<optab><mode>"
1495   [(match_operand:VI 0 "register_operand")
1496    (match_operand:<VM> 1 "vector_mask_operand")
1497    (any_int_unop:VI
1498      (match_operand:VI 2 "register_operand"))
1499    (match_operand:VI 3 "register_operand")]
1500   "TARGET_VECTOR"
1501 {
1502   /* Normalize into cond_len_* operations.  */
1503   emit_insn (gen_cond_len_<optab><mode> (operands[0], operands[1], operands[2],
1504                                          operands[3],
1505                                          gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1506                                          const0_rtx));
1507   DONE;
1508 })
1509
1510 (define_expand "cond_len_<optab><mode>"
1511   [(match_operand:VI 0 "register_operand")
1512    (match_operand:<VM> 1 "vector_mask_operand")
1513    (any_int_unop:VI
1514      (match_operand:VI 2 "register_operand"))
1515    (match_operand:VI 3 "register_operand")
1516    (match_operand 4 "autovec_length_operand")
1517    (match_operand 5 "const_0_operand")]
1518   "TARGET_VECTOR"
1519 {
1520   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1521   riscv_vector::expand_cond_len_unop (icode, operands);
1522   DONE;
1523 })
1524
1525 ;; -------------------------------------------------------------------------
1526 ;; ---- [FP] Conditional unary operations
1527 ;; -------------------------------------------------------------------------
1528 ;; Includes:
1529 ;; - vfneg/vfabs
1530 ;; -------------------------------------------------------------------------
1531
1532 (define_expand "cond_<optab><mode>"
1533   [(match_operand:VF 0 "register_operand")
1534    (match_operand:<VM> 1 "vector_mask_operand")
1535    (any_float_unop_nofrm:VF
1536      (match_operand:VF 2 "register_operand"))
1537    (match_operand:VF 3 "register_operand")]
1538   "TARGET_VECTOR"
1539 {
1540   /* Normalize into cond_len_* operations.  */
1541   emit_insn (gen_cond_len_<optab><mode> (operands[0], operands[1], operands[2],
1542                                          operands[3],
1543                                          gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1544                                          const0_rtx));
1545   DONE;
1546 })
1547
1548 (define_expand "cond_len_<optab><mode>"
1549   [(match_operand:VF 0 "register_operand")
1550    (match_operand:<VM> 1 "vector_mask_operand")
1551    (any_float_unop_nofrm:VF
1552      (match_operand:VF 2 "register_operand"))
1553    (match_operand:VF 3 "register_operand")
1554    (match_operand 4 "autovec_length_operand")
1555    (match_operand 5 "const_0_operand")]
1556   "TARGET_VECTOR"
1557 {
1558   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1559   riscv_vector::expand_cond_len_unop (icode, operands);
1560   DONE;
1561 })
1562
1563 ;; -------------------------------------------------------------------------
1564 ;; ---- [INT] Conditional binary operations
1565 ;; -------------------------------------------------------------------------
1566 ;; Includes:
1567 ;; - vsra/vsrl/vsll
1568 ;; -------------------------------------------------------------------------
1569
1570 (define_expand "cond_<optab><mode>"
1571   [(match_operand:VI 0 "register_operand")
1572    (match_operand:<VM> 1 "vector_mask_operand")
1573    (any_shift:VI
1574      (match_operand:VI 2 "register_operand")
1575      (match_operand:VI 3 "vector_shift_operand"))
1576    (match_operand:VI 4 "register_operand")]
1577   "TARGET_VECTOR"
1578 {
1579   /* Normalize into cond_len_* operations.  */
1580   emit_insn (gen_cond_len_<optab><mode> (operands[0], operands[1], operands[2],
1581                                          operands[3], operands[4],
1582                                          gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1583                                          const0_rtx));
1584   DONE;
1585 })
1586
1587 (define_expand "cond_len_<optab><mode>"
1588   [(match_operand:VI 0 "register_operand")
1589    (match_operand:<VM> 1 "vector_mask_operand")
1590    (any_shift:VI
1591      (match_operand:VI 2 "register_operand")
1592      (match_operand:VI 3 "vector_shift_operand"))
1593    (match_operand:VI 4 "register_operand")
1594    (match_operand 5 "autovec_length_operand")
1595    (match_operand 6 "const_0_operand")]
1596   "TARGET_VECTOR"
1597 {
1598   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1599   riscv_vector::expand_cond_len_binop (icode, operands);
1600   DONE;
1601 })
1602
1603 ;; -------------------------------------------------------------------------
1604 ;; ---- [INT] Conditional binary operations
1605 ;; -------------------------------------------------------------------------
1606 ;; Includes:
1607 ;; - vadd.vv/vsub.vv/...
1608 ;; - vadd.vi/vsub.vi/...
1609 ;; -------------------------------------------------------------------------
1610
1611 (define_expand "cond_<optab><mode>"
1612   [(match_operand:VI 0 "register_operand")
1613    (match_operand:<VM> 1 "vector_mask_operand")
1614    (any_int_binop_no_shift:VI
1615      (match_operand:VI 2 "<binop_rhs1_predicate>")
1616      (match_operand:VI 3 "<binop_rhs2_predicate>"))
1617    (match_operand:VI 4 "register_operand")]
1618   "TARGET_VECTOR"
1619 {
1620   /* Normalize into cond_len_* operations.  */
1621   emit_insn (gen_cond_len_<optab><mode> (operands[0], operands[1], operands[2],
1622                                          operands[3], operands[4],
1623                                          gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1624                                          const0_rtx));
1625   DONE;
1626 })
1627
1628 (define_expand "cond_len_<optab><mode>"
1629   [(match_operand:VI 0 "register_operand")
1630    (match_operand:<VM> 1 "vector_mask_operand")
1631    (any_int_binop_no_shift:VI
1632      (match_operand:VI 2 "<binop_rhs1_predicate>")
1633      (match_operand:VI 3 "<binop_rhs2_predicate>"))
1634    (match_operand:VI 4 "register_operand")
1635    (match_operand 5 "autovec_length_operand")
1636    (match_operand 6 "const_0_operand")]
1637   "TARGET_VECTOR"
1638 {
1639   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1640   riscv_vector::expand_cond_len_binop (icode, operands);
1641   DONE;
1642 })
1643
1644 ;; -------------------------------------------------------------------------
1645 ;; ---- [FP] Conditional binary operations
1646 ;; -------------------------------------------------------------------------
1647 ;; Includes:
1648 ;; - vfadd.vv/vfsub.vv/...
1649 ;; - vfadd.vf/vfsub.vf/...
1650 ;; -------------------------------------------------------------------------
1651
1652 (define_expand "cond_<optab><mode>"
1653   [(match_operand:VF 0 "register_operand")
1654    (match_operand:<VM> 1 "vector_mask_operand")
1655    (any_float_binop:VF
1656      (match_operand:VF 2 "register_operand")
1657      (match_operand:VF 3 "register_operand"))
1658    (match_operand:VF 4 "register_operand")]
1659   "TARGET_VECTOR"
1660 {
1661   /* Normalize into cond_len_* operations.  */
1662   emit_insn (gen_cond_len_<optab><mode> (operands[0], operands[1], operands[2],
1663                                          operands[3], operands[4],
1664                                          gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1665                                          const0_rtx));
1666   DONE;
1667 })
1668
1669 (define_expand "cond_len_<optab><mode>"
1670   [(match_operand:VF 0 "register_operand")
1671    (match_operand:<VM> 1 "vector_mask_operand")
1672    (any_float_binop:VF
1673      (match_operand:VF 2 "register_operand")
1674      (match_operand:VF 3 "register_operand"))
1675    (match_operand:VF 4 "register_operand")
1676    (match_operand 5 "autovec_length_operand")
1677    (match_operand 6 "const_0_operand")]
1678   "TARGET_VECTOR"
1679 {
1680   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1681   riscv_vector::expand_cond_len_binop (icode, operands);
1682   DONE;
1683 })
1684
1685 ;; -------------------------------------------------------------------------
1686 ;; Includes:
1687 ;; - vfmin.vv/vfmax.vv
1688 ;; - vfmin.vf/vfmax.vf
1689 ;; -------------------------------------------------------------------------
1690
1691 (define_expand "cond_<optab><mode>"
1692   [(match_operand:VF 0 "register_operand")
1693    (match_operand:<VM> 1 "vector_mask_operand")
1694    (any_float_binop_nofrm:VF
1695      (match_operand:VF 2 "register_operand")
1696      (match_operand:VF 3 "register_operand"))
1697    (match_operand:VF 4 "register_operand")]
1698   "TARGET_VECTOR"
1699 {
1700   /* Normalize into cond_len_* operations.  */
1701   emit_insn (gen_cond_len_<optab><mode> (operands[0], operands[1], operands[2],
1702                                          operands[3], operands[4],
1703                                          gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1704                                          const0_rtx));
1705   DONE;
1706 })
1707
1708 (define_expand "cond_len_<optab><mode>"
1709   [(match_operand:VF 0 "register_operand")
1710    (match_operand:<VM> 1 "vector_mask_operand")
1711    (any_float_binop_nofrm:VF
1712      (match_operand:VF 2 "register_operand")
1713      (match_operand:VF 3 "register_operand"))
1714    (match_operand:VF 4 "register_operand")
1715    (match_operand 5 "autovec_length_operand")
1716    (match_operand 6 "const_0_operand")]
1717   "TARGET_VECTOR"
1718 {
1719   insn_code icode = code_for_pred (<CODE>, <MODE>mode);
1720   riscv_vector::expand_cond_len_binop (icode, operands);
1721   DONE;
1722 })
1723
1724 ;; -------------------------------------------------------------------------
1725 ;; ---- [INT] Conditional ternary operations
1726 ;; -------------------------------------------------------------------------
1727 ;; Includes:
1728 ;; - vmacc/...
1729 ;; -------------------------------------------------------------------------
1730
1731 (define_expand "cond_fma<mode>"
1732   [(match_operand:VI 0 "register_operand")
1733    (match_operand:<VM> 1 "vector_mask_operand")
1734    (match_operand:VI 2 "register_operand")
1735    (match_operand:VI 3 "register_operand")
1736    (match_operand:VI 4 "register_operand")
1737    (match_operand:VI 5 "register_operand")]
1738   "TARGET_VECTOR"
1739 {
1740   /* Normalize into cond_len_* operations.  */
1741   emit_insn (gen_cond_len_fma<mode> (operands[0], operands[1], operands[2],
1742                                      operands[3], operands[4], operands[5],
1743                                      gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1744                                      const0_rtx));
1745   DONE;
1746 })
1747
1748 (define_expand "cond_len_fma<mode>"
1749   [(match_operand:VI 0 "register_operand")
1750    (match_operand:<VM> 1 "vector_mask_operand")
1751    (match_operand:VI 2 "register_operand")
1752    (match_operand:VI 3 "register_operand")
1753    (match_operand:VI 4 "register_operand")
1754    (match_operand:VI 5 "register_operand")
1755    (match_operand 6 "autovec_length_operand")
1756    (match_operand 7 "const_0_operand")]
1757   "TARGET_VECTOR"
1758 {
1759   insn_code icode = code_for_pred_mul_plus (<MODE>mode);
1760   riscv_vector::expand_cond_len_ternop (icode, operands);
1761   DONE;
1762 })
1763
1764 (define_expand "cond_fnma<mode>"
1765   [(match_operand:VI 0 "register_operand")
1766    (match_operand:<VM> 1 "vector_mask_operand")
1767    (match_operand:VI 2 "register_operand")
1768    (match_operand:VI 3 "register_operand")
1769    (match_operand:VI 4 "register_operand")
1770    (match_operand:VI 5 "register_operand")]
1771   "TARGET_VECTOR"
1772 {
1773   /* Normalize into cond_len_* operations.  */
1774   emit_insn (gen_cond_len_fnma<mode> (operands[0], operands[1], operands[2],
1775                                       operands[3], operands[4], operands[5],
1776                                       gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1777                                       const0_rtx));
1778   DONE;
1779 })
1780
1781 (define_expand "cond_len_fnma<mode>"
1782   [(match_operand:VI 0 "register_operand")
1783    (match_operand:<VM> 1 "vector_mask_operand")
1784    (match_operand:VI 2 "register_operand")
1785    (match_operand:VI 3 "register_operand")
1786    (match_operand:VI 4 "register_operand")
1787    (match_operand:VI 5 "register_operand")
1788    (match_operand 6 "autovec_length_operand")
1789    (match_operand 7 "const_0_operand")]
1790   "TARGET_VECTOR"
1791 {
1792   insn_code icode = code_for_pred_minus_mul (<MODE>mode);
1793   riscv_vector::expand_cond_len_ternop (icode, operands);
1794   DONE;
1795 })
1796
1797 ;; -------------------------------------------------------------------------
1798 ;; ---- [FP] Conditional ternary operations
1799 ;; -------------------------------------------------------------------------
1800 ;; Includes:
1801 ;; - vfmacc/...
1802 ;; -------------------------------------------------------------------------
1803
1804 (define_expand "cond_fma<mode>"
1805   [(match_operand:VF 0 "register_operand")
1806    (match_operand:<VM> 1 "vector_mask_operand")
1807    (match_operand:VF 2 "register_operand")
1808    (match_operand:VF 3 "register_operand")
1809    (match_operand:VF 4 "register_operand")
1810    (match_operand:VF 5 "register_operand")]
1811   "TARGET_VECTOR"
1812 {
1813   /* Normalize into cond_len_* operations.  */
1814   emit_insn (gen_cond_len_fma<mode> (operands[0], operands[1], operands[2],
1815                                      operands[3], operands[4], operands[5],
1816                                      gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1817                                      const0_rtx));
1818   DONE;
1819 })
1820
1821 (define_expand "cond_len_fma<mode>"
1822   [(match_operand:VF 0 "register_operand")
1823    (match_operand:<VM> 1 "vector_mask_operand")
1824    (match_operand:VF 2 "register_operand")
1825    (match_operand:VF 3 "register_operand")
1826    (match_operand:VF 4 "register_operand")
1827    (match_operand:VF 5 "register_operand")
1828    (match_operand 6 "autovec_length_operand")
1829    (match_operand 7 "const_0_operand")]
1830   "TARGET_VECTOR"
1831 {
1832   insn_code icode = code_for_pred_mul (PLUS, <MODE>mode);
1833   riscv_vector::expand_cond_len_ternop (icode, operands);
1834   DONE;
1835 })
1836
1837 (define_expand "cond_fnma<mode>"
1838   [(match_operand:VF 0 "register_operand")
1839    (match_operand:<VM> 1 "vector_mask_operand")
1840    (match_operand:VF 2 "register_operand")
1841    (match_operand:VF 3 "register_operand")
1842    (match_operand:VF 4 "register_operand")
1843    (match_operand:VF 5 "register_operand")]
1844   "TARGET_VECTOR"
1845 {
1846   /* Normalize into cond_len_* operations.  */
1847   emit_insn (gen_cond_len_fnma<mode> (operands[0], operands[1], operands[2],
1848                                       operands[3], operands[4], operands[5],
1849                                       gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1850                                       const0_rtx));
1851   DONE;
1852 })
1853
1854 (define_expand "cond_len_fnma<mode>"
1855   [(match_operand:VF 0 "register_operand")
1856    (match_operand:<VM> 1 "vector_mask_operand")
1857    (match_operand:VF 2 "register_operand")
1858    (match_operand:VF 3 "register_operand")
1859    (match_operand:VF 4 "register_operand")
1860    (match_operand:VF 5 "register_operand")
1861    (match_operand 6 "autovec_length_operand")
1862    (match_operand 7 "const_0_operand")]
1863   "TARGET_VECTOR"
1864 {
1865   insn_code icode = code_for_pred_mul_neg (PLUS, <MODE>mode);
1866   riscv_vector::expand_cond_len_ternop (icode, operands);
1867   DONE;
1868 })
1869
1870 (define_expand "cond_fms<mode>"
1871   [(match_operand:VF 0 "register_operand")
1872    (match_operand:<VM> 1 "vector_mask_operand")
1873    (match_operand:VF 2 "register_operand")
1874    (match_operand:VF 3 "register_operand")
1875    (match_operand:VF 4 "register_operand")
1876    (match_operand:VF 5 "register_operand")]
1877   "TARGET_VECTOR"
1878 {
1879   /* Normalize into cond_len_* operations.  */
1880   emit_insn (gen_cond_len_fms<mode> (operands[0], operands[1], operands[2],
1881                                      operands[3], operands[4], operands[5],
1882                                      gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1883                                      const0_rtx));
1884   DONE;
1885 })
1886
1887 (define_expand "cond_len_fms<mode>"
1888   [(match_operand:VF 0 "register_operand")
1889    (match_operand:<VM> 1 "vector_mask_operand")
1890    (match_operand:VF 2 "register_operand")
1891    (match_operand:VF 3 "register_operand")
1892    (match_operand:VF 4 "register_operand")
1893    (match_operand:VF 5 "register_operand")
1894    (match_operand 6 "autovec_length_operand")
1895    (match_operand 7 "const_0_operand")]
1896   "TARGET_VECTOR"
1897 {
1898   insn_code icode = code_for_pred_mul (MINUS, <MODE>mode);
1899   riscv_vector::expand_cond_len_ternop (icode, operands);
1900   DONE;
1901 })
1902
1903 (define_expand "cond_fnms<mode>"
1904   [(match_operand:VF 0 "register_operand")
1905    (match_operand:<VM> 1 "vector_mask_operand")
1906    (match_operand:VF 2 "register_operand")
1907    (match_operand:VF 3 "register_operand")
1908    (match_operand:VF 4 "register_operand")
1909    (match_operand:VF 5 "register_operand")]
1910   "TARGET_VECTOR"
1911 {
1912   /* Normalize into cond_len_* operations.  */
1913   emit_insn (gen_cond_len_fnms<mode> (operands[0], operands[1], operands[2],
1914                                       operands[3], operands[4], operands[5],
1915                                       gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode),
1916                                       const0_rtx));
1917   DONE;
1918 })
1919
1920 (define_expand "cond_len_fnms<mode>"
1921   [(match_operand:VF 0 "register_operand")
1922    (match_operand:<VM> 1 "vector_mask_operand")
1923    (match_operand:VF 2 "register_operand")
1924    (match_operand:VF 3 "register_operand")
1925    (match_operand:VF 4 "register_operand")
1926    (match_operand:VF 5 "register_operand")
1927    (match_operand 6 "autovec_length_operand")
1928    (match_operand 7 "const_0_operand")]
1929   "TARGET_VECTOR"
1930 {
1931   insn_code icode = code_for_pred_mul_neg (MINUS, <MODE>mode);
1932   riscv_vector::expand_cond_len_ternop (icode, operands);
1933   DONE;
1934 })
1935
1936 ;; =========================================================================
1937 ;; == Reductions
1938 ;; =========================================================================
1939
1940 ;; -------------------------------------------------------------------------
1941 ;; ---- [INT] Tree reductions
1942 ;; -------------------------------------------------------------------------
1943 ;; Includes:
1944 ;; - vredsum.vs
1945 ;; - vredmaxu.vs
1946 ;; - vredmax.vs
1947 ;; - vredminu.vs
1948 ;; - vredmin.vs
1949 ;; - vredand.vs
1950 ;; - vredor.vs
1951 ;; - vredxor.vs
1952 ;; -------------------------------------------------------------------------
1953
1954 (define_insn_and_split "reduc_plus_scal_<mode>"
1955   [(set (match_operand:<VEL> 0 "register_operand")
1956         (unspec:<VEL> [
1957              (match_operand:V_VLSI 1 "register_operand")
1958         ] UNSPEC_REDUC_SUM))]
1959   "TARGET_VECTOR && can_create_pseudo_p ()"
1960   "#"
1961   "&& 1"
1962   [(const_int 0)]
1963 {
1964   riscv_vector::expand_reduction (UNSPEC_REDUC_SUM, riscv_vector::REDUCE_OP,
1965                                   operands, CONST0_RTX (<VEL>mode));
1966   DONE;
1967 }
1968 [(set_attr "type" "vector")])
1969
1970 (define_expand "reduc_smax_scal_<mode>"
1971   [(match_operand:<VEL> 0 "register_operand")
1972    (match_operand:V_VLSI 1 "register_operand")]
1973   "TARGET_VECTOR"
1974 {
1975   int prec = GET_MODE_PRECISION (<VEL>mode);
1976   rtx min = immed_wide_int_const (wi::min_value (prec, SIGNED), <VEL>mode);
1977   riscv_vector::expand_reduction (UNSPEC_REDUC_MAX, riscv_vector::REDUCE_OP,
1978                                   operands, min);
1979   DONE;
1980 })
1981
1982 (define_expand "reduc_umax_scal_<mode>"
1983   [(match_operand:<VEL> 0 "register_operand")
1984    (match_operand:V_VLSI 1 "register_operand")]
1985   "TARGET_VECTOR"
1986 {
1987   riscv_vector::expand_reduction (UNSPEC_REDUC_MAXU, riscv_vector::REDUCE_OP,
1988                                   operands, CONST0_RTX (<VEL>mode));
1989   DONE;
1990 })
1991
1992 (define_expand "reduc_smin_scal_<mode>"
1993   [(match_operand:<VEL> 0 "register_operand")
1994    (match_operand:V_VLSI 1 "register_operand")]
1995   "TARGET_VECTOR"
1996 {
1997   int prec = GET_MODE_PRECISION (<VEL>mode);
1998   rtx max = immed_wide_int_const (wi::max_value (prec, SIGNED), <VEL>mode);
1999   riscv_vector::expand_reduction (UNSPEC_REDUC_MIN, riscv_vector::REDUCE_OP,
2000                                   operands, max);
2001   DONE;
2002 })
2003
2004 (define_expand "reduc_umin_scal_<mode>"
2005   [(match_operand:<VEL> 0 "register_operand")
2006    (match_operand:V_VLSI 1 "register_operand")]
2007   "TARGET_VECTOR"
2008 {
2009   int prec = GET_MODE_PRECISION (<VEL>mode);
2010   rtx max = immed_wide_int_const (wi::max_value (prec, UNSIGNED), <VEL>mode);
2011   riscv_vector::expand_reduction (UNSPEC_REDUC_MINU, riscv_vector::REDUCE_OP,
2012                                   operands, max);
2013   DONE;
2014 })
2015
2016 (define_expand "reduc_and_scal_<mode>"
2017   [(match_operand:<VEL> 0 "register_operand")
2018    (match_operand:V_VLSI 1 "register_operand")]
2019   "TARGET_VECTOR"
2020 {
2021   riscv_vector::expand_reduction (UNSPEC_REDUC_AND, riscv_vector::REDUCE_OP,
2022                                   operands, CONSTM1_RTX (<VEL>mode));
2023   DONE;
2024 })
2025
2026 (define_expand "reduc_ior_scal_<mode>"
2027   [(match_operand:<VEL> 0 "register_operand")
2028    (match_operand:V_VLSI 1 "register_operand")]
2029   "TARGET_VECTOR"
2030 {
2031   riscv_vector::expand_reduction (UNSPEC_REDUC_OR, riscv_vector::REDUCE_OP,
2032                                   operands, CONST0_RTX (<VEL>mode));
2033   DONE;
2034 })
2035
2036 (define_expand "reduc_xor_scal_<mode>"
2037   [(match_operand:<VEL> 0 "register_operand")
2038    (match_operand:V_VLSI 1 "register_operand")]
2039   "TARGET_VECTOR"
2040 {
2041   riscv_vector::expand_reduction (UNSPEC_REDUC_XOR, riscv_vector::REDUCE_OP,
2042                                   operands, CONST0_RTX (<VEL>mode));
2043   DONE;
2044 })
2045
2046 ;; -------------------------------------------------------------------------
2047 ;; ---- [FP] Tree reductions
2048 ;; -------------------------------------------------------------------------
2049 ;; Includes:
2050 ;; - vfredusum.vs
2051 ;; - vfredmax.vs
2052 ;; - vfredmin.vs
2053 ;; -------------------------------------------------------------------------
2054
2055 (define_insn_and_split "reduc_plus_scal_<mode>"
2056   [(set (match_operand:<VEL> 0 "register_operand")
2057         (unspec:<VEL> [
2058              (match_operand:V_VLSF 1 "register_operand")
2059         ] UNSPEC_REDUC_SUM_UNORDERED))]
2060   "TARGET_VECTOR && can_create_pseudo_p ()"
2061   "#"
2062   "&& 1"
2063   [(const_int 0)]
2064 {
2065   riscv_vector::expand_reduction (UNSPEC_REDUC_SUM_UNORDERED,
2066                                   riscv_vector::REDUCE_OP_FRM_DYN,
2067                                   operands, CONST0_RTX (<VEL>mode));
2068   DONE;
2069 }
2070 [(set_attr "type" "vector")])
2071
2072 (define_expand "reduc_smax_scal_<mode>"
2073   [(match_operand:<VEL> 0 "register_operand")
2074    (match_operand:V_VLSF 1 "register_operand")]
2075   "TARGET_VECTOR"
2076 {
2077   REAL_VALUE_TYPE rv;
2078   real_inf (&rv, true);
2079   rtx f = const_double_from_real_value (rv, <VEL>mode);
2080   riscv_vector::expand_reduction (UNSPEC_REDUC_MAX, riscv_vector::REDUCE_OP,
2081                                   operands, f);
2082   DONE;
2083 })
2084
2085 (define_expand "reduc_smin_scal_<mode>"
2086   [(match_operand:<VEL> 0 "register_operand")
2087    (match_operand:V_VLSF 1 "register_operand")]
2088   "TARGET_VECTOR"
2089 {
2090   REAL_VALUE_TYPE rv;
2091   real_inf (&rv, false);
2092   rtx f = const_double_from_real_value (rv, <VEL>mode);
2093   riscv_vector::expand_reduction (UNSPEC_REDUC_MIN, riscv_vector::REDUCE_OP,
2094                                   operands, f);
2095   DONE;
2096 })
2097
2098 ;; -------------------------------------------------------------------------
2099 ;; ---- [FP] Left-to-right reductions
2100 ;; -------------------------------------------------------------------------
2101 ;; Includes:
2102 ;; - vfredosum.vs
2103 ;; -------------------------------------------------------------------------
2104
2105 ;; Unpredicated in-order FP reductions.
2106 (define_insn_and_split "fold_left_plus_<mode>"
2107   [(set (match_operand:<VEL> 0 "register_operand")
2108         (unspec:<VEL> [
2109              (match_operand:V_VLSF 2 "register_operand")
2110              (match_operand:<VEL> 1 "register_operand")
2111         ] UNSPEC_REDUC_SUM_ORDERED))]
2112   "TARGET_VECTOR && can_create_pseudo_p ()"
2113   "#"
2114   "&& 1"
2115   [(const_int 0)]
2116 {
2117   rtx ops[] = {operands[0], operands[2]};
2118   riscv_vector::expand_reduction (UNSPEC_REDUC_SUM_ORDERED,
2119                                   riscv_vector::REDUCE_OP_FRM_DYN,
2120                                   ops, operands[1]);
2121   DONE;
2122 }
2123 [(set_attr "type" "vector")])
2124
2125 ;; Predicated in-order FP reductions.
2126 (define_insn_and_split "mask_len_fold_left_plus_<mode>"
2127   [(set (match_operand:<VEL> 0 "register_operand")
2128         (unspec:<VEL> [
2129           (match_operand:VF 2 "register_operand")
2130           (match_operand:<VEL> 1 "register_operand")
2131           (match_operand:<VM> 3 "vector_mask_operand")
2132           (match_operand 4 "autovec_length_operand")
2133           (match_operand 5 "const_0_operand")
2134         ] UNSPEC_REDUC_SUM_ORDERED))]
2135   "TARGET_VECTOR && can_create_pseudo_p ()"
2136   "#"
2137   "&& 1"
2138   [(const_int 0)]
2139 {
2140   if (rtx_equal_p (operands[4], const0_rtx))
2141     emit_move_insn (operands[0], operands[1]);
2142   else
2143     {
2144       rtx ops[] = {operands[0], operands[2], operands[3], operands[4]};
2145       riscv_vector::expand_reduction (UNSPEC_REDUC_SUM_ORDERED,
2146                                       riscv_vector::REDUCE_OP_M_FRM_DYN,
2147                                       ops, operands[1]);
2148     }
2149   DONE;
2150 }
2151 [(set_attr "type" "vector")])
2152
2153 ;; -------------------------------------------------------------------------
2154 ;; ---- [INT,FP] Extract active element
2155 ;; -------------------------------------------------------------------------
2156 ;; Includes:
2157 ;; - vcompress.vm
2158 ;; - vcpop.m
2159 ;; - vslidedown.vx
2160 ;; - vmv.x.s
2161 ;; - vfmv.f.s
2162 ;; -------------------------------------------------------------------------
2163
2164 (define_expand "len_fold_extract_last_<mode>"
2165   [(match_operand:<VEL> 0 "register_operand")
2166    (match_operand:<VEL> 1 "register_operand")
2167    (match_operand:<VM> 2 "register_operand")
2168    (match_operand:V 3 "register_operand")
2169    (match_operand 4 "autovec_length_operand")
2170    (match_operand 5 "const_0_operand")]
2171   "TARGET_VECTOR"
2172   {
2173     riscv_vector::expand_fold_extract_last (operands);
2174     DONE;
2175   })
2176
2177 ;; -------------------------------------------------------------------------
2178 ;; ---- [INT] Average.
2179 ;; -------------------------------------------------------------------------
2180 ;; Implements the following "average" patterns:
2181 ;; floor:
2182 ;;  op[0] = (narrow) ((wide) op[1] + (wide) op[2]) >> 1;
2183 ;; ceil:
2184 ;;  op[0] = (narrow) ((wide) op[1] + (wide) op[2] + 1)) >> 1;
2185 ;; -------------------------------------------------------------------------
2186
2187 (define_expand "<u>avg<v_double_trunc>3_floor"
2188  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
2189    (truncate:<V_DOUBLE_TRUNC>
2190     (<ext_to_rshift>:VWEXTI
2191      (plus:VWEXTI
2192       (any_extend:VWEXTI
2193        (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
2194       (any_extend:VWEXTI
2195        (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))))]
2196   "TARGET_VECTOR"
2197 {
2198   /* First emit a widening addition.  */
2199   rtx tmp1 = gen_reg_rtx (<MODE>mode);
2200   rtx ops1[] = {tmp1, operands[1], operands[2]};
2201   insn_code icode = code_for_pred_dual_widen (PLUS, <CODE>, <MODE>mode);
2202   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops1);
2203
2204   /* Then a narrowing shift.  */
2205   rtx ops2[] = {operands[0], tmp1, const1_rtx};
2206   icode = code_for_pred_narrow_scalar (<EXT_TO_RSHIFT>, <MODE>mode);
2207   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops2);
2208   DONE;
2209 })
2210
2211 (define_expand "<u>avg<v_double_trunc>3_ceil"
2212  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
2213    (truncate:<V_DOUBLE_TRUNC>
2214     (<ext_to_rshift>:VWEXTI
2215      (plus:VWEXTI
2216       (plus:VWEXTI
2217        (any_extend:VWEXTI
2218         (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
2219        (any_extend:VWEXTI
2220         (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
2221       (const_int 1)))))]
2222   "TARGET_VECTOR"
2223 {
2224   /* First emit a widening addition.  */
2225   rtx tmp1 = gen_reg_rtx (<MODE>mode);
2226   rtx ops1[] = {tmp1, operands[1], operands[2]};
2227   insn_code icode = code_for_pred_dual_widen (PLUS, <CODE>, <MODE>mode);
2228   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops1);
2229
2230   /* Then add 1.  */
2231   rtx tmp2 = gen_reg_rtx (<MODE>mode);
2232   rtx ops2[] = {tmp2, tmp1, const1_rtx};
2233   icode = code_for_pred_scalar (PLUS, <MODE>mode);
2234   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops2);
2235
2236   /* Finally, a narrowing shift.  */
2237   rtx ops3[] = {operands[0], tmp2, const1_rtx};
2238   icode = code_for_pred_narrow_scalar (<EXT_TO_RSHIFT>, <MODE>mode);
2239   riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops3);
2240   DONE;
2241 })