gcc/config/riscv/vector.md

   1 ;; Machine description for RISC-V 'V' Extension for GNU compiler.
   2 ;; Copyright (C) 2022-2023 Free Software Foundation, Inc.
   3 ;; Contributed by Juzhe Zhong (juzhe.zhong@rivai.ai), RiVAI Technologies Ltd.
   4
   5 ;; This file is part of GCC.
   6
   7 ;; GCC is free software; you can redistribute it and/or modify
   8 ;; it under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11
  12 ;; GCC is distributed in the hope that it will be useful,
  13 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;; GNU General Public License for more details.
  16
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; This file describes the RISC-V 'V' Extension, Version 1.0.
  22 ;;
  23 ;; This file include :
  24 ;;
  25 ;; - Intrinsics (https://github.com/riscv/rvv-intrinsic-doc)
  26 ;; - Auto-vectorization (TBD)
  27 ;; - Combine optimization (TBD)
  28
  29 (include "vector-iterators.md")
  30
  31 (define_constants [
  32    (INVALID_ATTRIBUTE            255)
  33    (X0_REGNUM                      0)
  34 ])
  35
  36 ;; True if the type is RVV instructions that include VTYPE
  37 ;; global status register in the use op list.
  38 ;; We known VTYPE has 4 fields: SEW, LMUL, TA, MA.
  39 ;; The instruction need any of VTYPE field is set as true
  40 ;; in this attribute.
  41 (define_attr "has_vtype_op" "false,true"
  42   (cond [(eq_attr "type" "vlde,vste,vldm,vstm,vlds,vsts,\
  43                           vldux,vldox,vstux,vstox,vldff,\
  44                           vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,\
  45                           vimul,vidiv,viwmul,vimuladd,viwmuladd,vimerge,vimov,\
  46                           vsalu,vaalu,vsmul,vsshift,vnclip,\
  47                           vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,\
  48                           vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,\
  49                           vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,\
  50                           vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,\
  51                           vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
  52                           vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,vfmovvf,vfmovfv,\
  53                           vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
  54                           vgather,vcompress")
  55          (const_string "true")]
  56         (const_string "false")))
  57
  58 ;; True if the type is RVV instructions that include VL
  59 ;; global status register in the use op list.
  60 ;; The instruction need vector length to be specified is set
  61 ;; in this attribute.
  62 (define_attr "has_vl_op" "false,true"
  63   (cond [(eq_attr "type" "vlde,vste,vldm,vstm,vlds,vsts,\
  64                           vldux,vldox,vstux,vstox,vldff,\
  65                           vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,\
  66                           vimul,vidiv,viwmul,vimuladd,viwmuladd,vimerge,vimov,\
  67                           vsalu,vaalu,vsmul,vsshift,vnclip,\
  68                           vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,\
  69                           vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,\
  70                           vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,\
  71                           vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,\
  72                           vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
  73                           vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovxv,vfmovfv,\
  74                           vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
  75                           vgather,vcompress")
  76          (const_string "true")]
  77         (const_string "false")))
  78
  79 ;; The default SEW of RVV instruction. This attribute doesn't mean the instruction
  80 ;; is necessary to require SEW check for example vlm.v which require ratio to
  81 ;; check. However, we need default value of SEW for vsetvl instruction since there
  82 ;; is no field for ratio in the vsetvl instruction encoding.
  83 (define_attr "sew" ""
  84   (cond [(eq_attr "mode" "VNx1QI,VNx2QI,VNx4QI,VNx8QI,VNx16QI,VNx32QI,VNx64QI,\
  85                           VNx1BI,VNx2BI,VNx4BI,VNx8BI,VNx16BI,VNx32BI,VNx64BI")
  86          (const_int 8)
  87          (eq_attr "mode" "VNx1HI,VNx2HI,VNx4HI,VNx8HI,VNx16HI,VNx32HI")
  88          (const_int 16)
  89          (eq_attr "mode" "VNx1SI,VNx2SI,VNx4SI,VNx8SI,VNx16SI,\
  90                           VNx1SF,VNx2SF,VNx4SF,VNx8SF,VNx16SF")
  91          (const_int 32)
  92          (eq_attr "mode" "VNx1DI,VNx2DI,VNx4DI,VNx8DI,\
  93                           VNx1DF,VNx2DF,VNx4DF,VNx8DF")
  94          (const_int 64)]
  95         (const_int INVALID_ATTRIBUTE)))
  96
  97 ;; Ditto to LMUL.
  98 (define_attr "vlmul" ""
  99   (cond [(eq_attr "mode" "VNx1QI,VNx1BI")
 100            (symbol_ref "riscv_vector::get_vlmul(E_VNx1QImode)")
 101          (eq_attr "mode" "VNx2QI,VNx2BI")
 102            (symbol_ref "riscv_vector::get_vlmul(E_VNx2QImode)")
 103          (eq_attr "mode" "VNx4QI,VNx4BI")
 104            (symbol_ref "riscv_vector::get_vlmul(E_VNx4QImode)")
 105          (eq_attr "mode" "VNx8QI,VNx8BI")
 106            (symbol_ref "riscv_vector::get_vlmul(E_VNx8QImode)")
 107          (eq_attr "mode" "VNx16QI,VNx16BI")
 108            (symbol_ref "riscv_vector::get_vlmul(E_VNx16QImode)")
 109          (eq_attr "mode" "VNx32QI,VNx32BI")
 110            (symbol_ref "riscv_vector::get_vlmul(E_VNx32QImode)")
 111          (eq_attr "mode" "VNx64QI,VNx64BI")
 112            (symbol_ref "riscv_vector::get_vlmul(E_VNx64QImode)")
 113          (eq_attr "mode" "VNx1HI")
 114            (symbol_ref "riscv_vector::get_vlmul(E_VNx1HImode)")
 115          (eq_attr "mode" "VNx2HI")
 116            (symbol_ref "riscv_vector::get_vlmul(E_VNx2HImode)")
 117          (eq_attr "mode" "VNx4HI")
 118            (symbol_ref "riscv_vector::get_vlmul(E_VNx4HImode)")
 119          (eq_attr "mode" "VNx8HI")
 120            (symbol_ref "riscv_vector::get_vlmul(E_VNx8HImode)")
 121          (eq_attr "mode" "VNx16HI")
 122            (symbol_ref "riscv_vector::get_vlmul(E_VNx16HImode)")
 123          (eq_attr "mode" "VNx32HI")
 124            (symbol_ref "riscv_vector::get_vlmul(E_VNx32HImode)")
 125          (eq_attr "mode" "VNx1SI,VNx1SF")
 126            (symbol_ref "riscv_vector::get_vlmul(E_VNx1SImode)")
 127          (eq_attr "mode" "VNx2SI,VNx2SF")
 128            (symbol_ref "riscv_vector::get_vlmul(E_VNx2SImode)")
 129          (eq_attr "mode" "VNx4SI,VNx4SF")
 130            (symbol_ref "riscv_vector::get_vlmul(E_VNx4SImode)")
 131          (eq_attr "mode" "VNx8SI,VNx8SF")
 132            (symbol_ref "riscv_vector::get_vlmul(E_VNx8SImode)")
 133          (eq_attr "mode" "VNx16SI,VNx16SF")
 134            (symbol_ref "riscv_vector::get_vlmul(E_VNx16SImode)")
 135          (eq_attr "mode" "VNx1DI,VNx1DF")
 136            (symbol_ref "riscv_vector::get_vlmul(E_VNx1DImode)")
 137          (eq_attr "mode" "VNx2DI,VNx2DF")
 138            (symbol_ref "riscv_vector::get_vlmul(E_VNx2DImode)")
 139          (eq_attr "mode" "VNx4DI,VNx4DF")
 140            (symbol_ref "riscv_vector::get_vlmul(E_VNx4DImode)")
 141          (eq_attr "mode" "VNx8DI,VNx8DF")
 142            (symbol_ref "riscv_vector::get_vlmul(E_VNx8DImode)")]
 143         (const_int INVALID_ATTRIBUTE)))
 144
 145 ;; It is valid for instruction that require sew/lmul ratio.
 146 (define_attr "ratio" ""
 147   (cond [(eq_attr "type" "vimov,vfmov,vldux,vldox,vstux,vstox,\
 148                           vialu,vshift,vicmp,vimul,vidiv,vsalu,\
 149                           vext,viwalu,viwmul,vicalu,vnshift,\
 150                           vimuladd,vimerge,vaalu,vsmul,vsshift,\
 151                           vnclip,viminmax,viwmuladd,vmpop,vmffs,vmsfs,\
 152                           vmiota,vmidx,vfalu,vfmul,vfminmax,vfdiv,\
 153                           vfwalu,vfwmul,vfsqrt,vfrecp,vfsgnj,vfcmp,\
 154                           vfmerge,vfcvtitof,vfcvtftoi,vfwcvtitof,\
 155                           vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,\
 156                           vfncvtftof,vfmuladd,vfwmuladd,vfclass,vired,\
 157                           viwred,vfredu,vfredo,vfwredu,vfwredo,vimovvx,\
 158                           vimovxv,vfmovvf,vfmovfv,vslideup,vslidedown,\
 159                           vislide1up,vislide1down,vfslide1up,vfslide1down,\
 160                           vgather,vcompress")
 161            (const_int INVALID_ATTRIBUTE)
 162          (eq_attr "mode" "VNx1QI,VNx1BI")
 163            (symbol_ref "riscv_vector::get_ratio(E_VNx1QImode)")
 164          (eq_attr "mode" "VNx2QI,VNx2BI")
 165            (symbol_ref "riscv_vector::get_ratio(E_VNx2QImode)")
 166          (eq_attr "mode" "VNx4QI,VNx4BI")
 167            (symbol_ref "riscv_vector::get_ratio(E_VNx4QImode)")
 168          (eq_attr "mode" "VNx8QI,VNx8BI")
 169            (symbol_ref "riscv_vector::get_ratio(E_VNx8QImode)")
 170          (eq_attr "mode" "VNx16QI,VNx16BI")
 171            (symbol_ref "riscv_vector::get_ratio(E_VNx16QImode)")
 172          (eq_attr "mode" "VNx32QI,VNx32BI")
 173            (symbol_ref "riscv_vector::get_ratio(E_VNx32QImode)")
 174          (eq_attr "mode" "VNx64QI,VNx64BI")
 175            (symbol_ref "riscv_vector::get_ratio(E_VNx64QImode)")
 176          (eq_attr "mode" "VNx1HI")
 177            (symbol_ref "riscv_vector::get_ratio(E_VNx1HImode)")
 178          (eq_attr "mode" "VNx2HI")
 179            (symbol_ref "riscv_vector::get_ratio(E_VNx2HImode)")
 180          (eq_attr "mode" "VNx4HI")
 181            (symbol_ref "riscv_vector::get_ratio(E_VNx4HImode)")
 182          (eq_attr "mode" "VNx8HI")
 183            (symbol_ref "riscv_vector::get_ratio(E_VNx8HImode)")
 184          (eq_attr "mode" "VNx16HI")
 185            (symbol_ref "riscv_vector::get_ratio(E_VNx16HImode)")
 186          (eq_attr "mode" "VNx32HI")
 187            (symbol_ref "riscv_vector::get_ratio(E_VNx32HImode)")
 188          (eq_attr "mode" "VNx1SI,VNx1SF")
 189            (symbol_ref "riscv_vector::get_ratio(E_VNx1SImode)")
 190          (eq_attr "mode" "VNx2SI,VNx2SF")
 191            (symbol_ref "riscv_vector::get_ratio(E_VNx2SImode)")
 192          (eq_attr "mode" "VNx4SI,VNx4SF")
 193            (symbol_ref "riscv_vector::get_ratio(E_VNx4SImode)")
 194          (eq_attr "mode" "VNx8SI,VNx8SF")
 195            (symbol_ref "riscv_vector::get_ratio(E_VNx8SImode)")
 196          (eq_attr "mode" "VNx16SI,VNx16SF")
 197            (symbol_ref "riscv_vector::get_ratio(E_VNx16SImode)")
 198          (eq_attr "mode" "VNx1DI,VNx1DF")
 199            (symbol_ref "riscv_vector::get_ratio(E_VNx1DImode)")
 200          (eq_attr "mode" "VNx2DI,VNx2DF")
 201            (symbol_ref "riscv_vector::get_ratio(E_VNx2DImode)")
 202          (eq_attr "mode" "VNx4DI,VNx4DF")
 203            (symbol_ref "riscv_vector::get_ratio(E_VNx4DImode)")
 204          (eq_attr "mode" "VNx8DI,VNx8DF")
 205            (symbol_ref "riscv_vector::get_ratio(E_VNx8DImode)")]
 206         (const_int INVALID_ATTRIBUTE)))
 207
 208 ;; The index of operand[] to get the merge op.
 209 (define_attr "merge_op_idx" ""
 210         (cond [(eq_attr "type" "vlde,vimov,vfmov,vldm,vlds,vmalu,vldux,vldox,vicmp,\
 211                                 vialu,vshift,viminmax,vimul,vidiv,vsalu,vext,viwalu,\
 212                                 viwmul,vnshift,vaalu,vsmul,vsshift,vnclip,vmsfs,\
 213                                 vmiota,vmidx,vfalu,vfmul,vfminmax,vfdiv,vfwalu,vfwmul,\
 214                                 vfsqrt,vfrecp,vfsgnj,vfcmp,vfcvtitof,vfcvtftoi,vfwcvtitof,\
 215                                 vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,vfclass,\
 216                                 vired,viwred,vfredu,vfredo,vfwredu,vfwredo,vimovxv,vfmovfv,\
 217                                 vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
 218                                 vgather,vldff,viwmuladd,vfwmuladd")
 219                (const_int 2)
 220
 221                (eq_attr "type" "vimerge,vfmerge,vcompress")
 222                (const_int 1)
 223
 224                (eq_attr "type" "vimuladd,vfmuladd")
 225                (const_int 5)]
 226         (const_int INVALID_ATTRIBUTE)))
 227
 228 ;; The index of operand[] to get the avl op.
 229 (define_attr "vl_op_idx" ""
 230   (cond [(eq_attr "type" "vlde,vste,vimov,vfmov,vldm,vstm,vmalu,vsts,vstux,\
 231                           vstox,vext,vmsfs,vmiota,vfsqrt,vfrecp,vfcvtitof,vldff,\
 232                           vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,\
 233                           vfncvtftoi,vfncvtftof,vfclass,vimovxv,vfmovfv,vcompress")
 234            (const_int 4)
 235
 236          ;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
 237          ;; wheras it is pred_strided_load if operands[3] is vector mode.
 238          (eq_attr "type" "vlds")
 239            (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
 240              (const_int 5)
 241              (const_int 4))
 242
 243          (eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
 244                           viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\
 245                           vsshift,vnclip,vfalu,vfmul,vfminmax,vfdiv,vfwalu,vfwmul,\
 246                           vfsgnj,vfmerge,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
 247                           vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
 248                           vgather,viwmuladd,vfwmuladd")
 249            (const_int 5)
 250
 251          (eq_attr "type" "vicmp,vimuladd,vfcmp,vfmuladd")
 252            (const_int 6)
 253
 254          (eq_attr "type" "vmpop,vmffs,vmidx")
 255            (const_int 3)]
 256   (const_int INVALID_ATTRIBUTE)))
 257
 258 ;; The tail policy op value.
 259 (define_attr "ta" ""
 260   (cond [(eq_attr "type" "vlde,vimov,vfmov,vext,vmiota,vfsqrt,vfrecp,\
 261                           vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,\
 262                           vfncvtitof,vfncvtftoi,vfncvtftof,vfclass,vimovxv,vfmovfv,\
 263                           vcompress,vldff")
 264            (symbol_ref "riscv_vector::get_ta(operands[5])")
 265
 266          ;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
 267          ;; wheras it is pred_strided_load if operands[3] is vector mode.
 268          (eq_attr "type" "vlds")
 269            (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
 270              (symbol_ref "riscv_vector::get_ta(operands[6])")
 271              (symbol_ref "riscv_vector::get_ta(operands[5])"))
 272
 273          (eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
 274                           viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\
 275                           vsshift,vnclip,vfalu,vfmul,vfminmax,vfdiv,\
 276                           vfwalu,vfwmul,vfsgnj,vfmerge,vired,viwred,vfredu,\
 277                           vfredo,vfwredu,vfwredo,vslideup,vslidedown,vislide1up,\
 278                           vislide1down,vfslide1up,vfslide1down,vgather,viwmuladd,vfwmuladd")
 279            (symbol_ref "riscv_vector::get_ta(operands[6])")
 280
 281          (eq_attr "type" "vimuladd,vfmuladd")
 282            (symbol_ref "riscv_vector::get_ta(operands[7])")
 283
 284          (eq_attr "type" "vmidx")
 285            (symbol_ref "riscv_vector::get_ta(operands[4])")]
 286         (const_int INVALID_ATTRIBUTE)))
 287
 288 ;; The mask policy op value.
 289 (define_attr "ma" ""
 290   (cond [(eq_attr "type" "vlde,vext,vmiota,vfsqrt,vfrecp,vfcvtitof,vfcvtftoi,\
 291                           vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,\
 292                           vfncvtftof,vfclass,vldff")
 293            (symbol_ref "riscv_vector::get_ma(operands[6])")
 294
 295          ;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
 296          ;; wheras it is pred_strided_load if operands[3] is vector mode.
 297          (eq_attr "type" "vlds")
 298            (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
 299              (symbol_ref "riscv_vector::get_ma(operands[7])")
 300              (symbol_ref "riscv_vector::get_ma(operands[6])"))
 301
 302          (eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
 303                           viwalu,viwmul,vnshift,vaalu,vsmul,vsshift,\
 304                           vnclip,vicmp,vfalu,vfmul,vfminmax,vfdiv,\
 305                           vfwalu,vfwmul,vfsgnj,vfcmp,vslideup,vslidedown,\
 306                           vislide1up,vislide1down,vfslide1up,vfslide1down,vgather,\
 307                           viwmuladd,vfwmuladd")
 308            (symbol_ref "riscv_vector::get_ma(operands[7])")
 309
 310          (eq_attr "type" "vimuladd,vfmuladd")
 311            (symbol_ref "riscv_vector::get_ma(operands[8])")
 312
 313          (eq_attr "type" "vmsfs,vmidx")
 314            (symbol_ref "riscv_vector::get_ma(operands[5])")]
 315         (const_int INVALID_ATTRIBUTE)))
 316
 317 ;; The avl type value.
 318 (define_attr "avl_type" ""
 319   (cond [(eq_attr "type" "vlde,vldff,vste,vimov,vimov,vimov,vfmov,vext,vimerge,\
 320                           vfsqrt,vfrecp,vfmerge,vfcvtitof,vfcvtftoi,vfwcvtitof,\
 321                           vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,\
 322                           vfclass,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
 323                           vimovxv,vfmovfv")
 324            (symbol_ref "INTVAL (operands[7])")
 325          (eq_attr "type" "vldm,vstm,vimov,vmalu,vmalu")
 326            (symbol_ref "INTVAL (operands[5])")
 327
 328          ;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
 329          ;; wheras it is pred_strided_load if operands[3] is vector mode.
 330          (eq_attr "type" "vlds")
 331            (if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
 332              (const_int INVALID_ATTRIBUTE)
 333              (symbol_ref "INTVAL (operands[7])"))
 334
 335          (eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
 336                           viwalu,viwmul,vnshift,vimuladd,vaalu,vsmul,vsshift,\
 337                           vnclip,vicmp,vfalu,vfmul,vfminmax,vfdiv,vfwalu,vfwmul,\
 338                           vfsgnj,vfcmp,vfmuladd,vslideup,vslidedown,vislide1up,\
 339                           vislide1down,vfslide1up,vfslide1down,vgather,viwmuladd,vfwmuladd")
 340            (symbol_ref "INTVAL (operands[8])")
 341          (eq_attr "type" "vstux,vstox")
 342            (symbol_ref "INTVAL (operands[5])")
 343
 344          (eq_attr "type" "vimuladd")
 345            (symbol_ref "INTVAL (operands[9])")
 346
 347          (eq_attr "type" "vmsfs,vmidx,vcompress")
 348            (symbol_ref "INTVAL (operands[6])")
 349
 350          (eq_attr "type" "vmpop,vmffs")
 351            (symbol_ref "INTVAL (operands[4])")]
 352         (const_int INVALID_ATTRIBUTE)))
 353
 354 ;; -----------------------------------------------------------------
 355 ;; ---- Miscellaneous Operations
 356 ;; -----------------------------------------------------------------
 357
 358 (define_insn "@vundefined<mode>"
 359   [(set (match_operand:V 0 "register_operand" "=vr")
 360         (unspec:V [(reg:SI X0_REGNUM)] UNSPEC_VUNDEF))]
 361   "TARGET_VECTOR"
 362   "")
 363
 364 (define_insn "@vundefined<mode>"
 365   [(set (match_operand:VB 0 "register_operand" "=vr")
 366         (unspec:VB [(reg:SI X0_REGNUM)] UNSPEC_VUNDEF))]
 367   "TARGET_VECTOR"
 368   "")
 369
 370 (define_expand "@vreinterpret<mode>"
 371   [(set (match_operand:V 0 "register_operand")
 372         (match_operand 1 "vector_any_register_operand"))]
 373   "TARGET_VECTOR"
 374   {
 375     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, operands[1]));
 376     DONE;
 377   }
 378 )
 379
 380 (define_expand "@vlmul_extx2<mode>"
 381   [(set (match_operand:<VLMULX2> 0 "register_operand")
 382         (subreg:<VLMULX2>
 383           (match_operand:VLMULEXT2 1 "register_operand") 0))]
 384   "TARGET_VECTOR"
 385 {})
 386
 387 (define_expand "@vlmul_extx4<mode>"
 388   [(set (match_operand:<VLMULX4> 0 "register_operand")
 389         (subreg:<VLMULX4>
 390           (match_operand:VLMULEXT4 1 "register_operand") 0))]
 391   "TARGET_VECTOR"
 392 {})
 393
 394 (define_expand "@vlmul_extx8<mode>"
 395   [(set (match_operand:<VLMULX8> 0 "register_operand")
 396         (subreg:<VLMULX8>
 397           (match_operand:VLMULEXT8 1 "register_operand") 0))]
 398   "TARGET_VECTOR"
 399 {})
 400
 401 (define_expand "@vlmul_extx16<mode>"
 402   [(set (match_operand:<VLMULX16> 0 "register_operand")
 403         (subreg:<VLMULX16>
 404           (match_operand:VLMULEXT16 1 "register_operand") 0))]
 405   "TARGET_VECTOR"
 406 {})
 407
 408 (define_expand "@vlmul_extx32<mode>"
 409   [(set (match_operand:<VLMULX32> 0 "register_operand")
 410         (subreg:<VLMULX32>
 411           (match_operand:VLMULEXT32 1 "register_operand") 0))]
 412   "TARGET_VECTOR"
 413 {})
 414
 415 (define_expand "@vlmul_extx64<mode>"
 416   [(set (match_operand:<VLMULX64> 0 "register_operand")
 417         (subreg:<VLMULX64>
 418           (match_operand:VLMULEXT64 1 "register_operand") 0))]
 419   "TARGET_VECTOR"
 420 {})
 421
 422 (define_insn_and_split "*vlmul_extx2<mode>"
 423   [(set (match_operand:<VLMULX2> 0 "register_operand"  "=vr, ?&vr")
 424         (subreg:<VLMULX2>
 425           (match_operand:VLMULEXT2 1 "register_operand" " 0,   vr") 0))]
 426   "TARGET_VECTOR"
 427   "#"
 428   "&& reload_completed"
 429   [(const_int 0)]
 430 {
 431   emit_insn (gen_rtx_SET (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
 432   DONE;
 433 })
 434
 435 (define_insn_and_split "*vlmul_extx4<mode>"
 436   [(set (match_operand:<VLMULX4> 0 "register_operand"  "=vr, ?&vr")
 437         (subreg:<VLMULX4>
 438           (match_operand:VLMULEXT4 1 "register_operand" " 0,   vr") 0))]
 439   "TARGET_VECTOR"
 440   "#"
 441   "&& reload_completed"
 442   [(const_int 0)]
 443 {
 444   emit_insn (gen_rtx_SET (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
 445   DONE;
 446 })
 447
 448 (define_insn_and_split "*vlmul_extx8<mode>"
 449   [(set (match_operand:<VLMULX8> 0 "register_operand"  "=vr, ?&vr")
 450         (subreg:<VLMULX8>
 451           (match_operand:VLMULEXT8 1 "register_operand" " 0,   vr") 0))]
 452   "TARGET_VECTOR"
 453   "#"
 454   "&& reload_completed"
 455   [(const_int 0)]
 456 {
 457   emit_insn (gen_rtx_SET (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
 458   DONE;
 459 })
 460
 461 (define_insn_and_split "*vlmul_extx16<mode>"
 462   [(set (match_operand:<VLMULX16> 0 "register_operand"  "=vr, ?&vr")
 463         (subreg:<VLMULX16>
 464           (match_operand:VLMULEXT16 1 "register_operand" " 0,   vr") 0))]
 465   "TARGET_VECTOR"
 466   "#"
 467   "&& reload_completed"
 468   [(const_int 0)]
 469 {
 470   emit_insn (gen_rtx_SET (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
 471   DONE;
 472 })
 473
 474 (define_insn_and_split "*vlmul_extx32<mode>"
 475   [(set (match_operand:<VLMULX32> 0 "register_operand"  "=vr, ?&vr")
 476         (subreg:<VLMULX32>
 477           (match_operand:VLMULEXT32 1 "register_operand" " 0,   vr") 0))]
 478   "TARGET_VECTOR"
 479   "#"
 480   "&& reload_completed"
 481   [(const_int 0)]
 482 {
 483   emit_insn (gen_rtx_SET (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
 484   DONE;
 485 })
 486
 487 (define_insn_and_split "*vlmul_extx64<mode>"
 488   [(set (match_operand:<VLMULX64> 0 "register_operand"  "=vr, ?&vr")
 489         (subreg:<VLMULX64>
 490           (match_operand:VLMULEXT64 1 "register_operand" " 0,   vr") 0))]
 491   "TARGET_VECTOR"
 492   "#"
 493   "&& reload_completed"
 494   [(const_int 0)]
 495 {
 496   emit_insn (gen_rtx_SET (gen_lowpart (<MODE>mode, operands[0]), operands[1]));
 497   DONE;
 498 })
 499
 500 ;; This pattern is used to hold the AVL operand for
 501 ;; RVV instructions that implicity use VLMAX AVL.
 502 ;; RVV instruction implicitly use GPR that is ultimately
 503 ;; defined by this pattern is safe for VSETVL pass emit
 504 ;; a vsetvl instruction modify this register after RA.
 505 ;; Case 1:
 506 ;;   vlmax_avl a5
 507 ;;   ... (across many blocks)
 508 ;;   vadd (implicit use a5)  ====> emit: vsetvl a5,zero
 509 ;; Case 2:
 510 ;;   vlmax_avl a5
 511 ;;   ... (across many blocks)
 512 ;;   mv a6,a5
 513 ;;   ... (across many blocks)
 514 ;;   vadd (implicit use a6)  ====> emit: vsetvl a6,zero
 515 ;; Case 3:
 516 ;;   vlmax_avl a5
 517 ;;   ... (across many blocks)
 518 ;;   store mem,a5 (spill)
 519 ;;   ... (across many blocks)
 520 ;;   load a7,mem (spill)
 521 ;;   ... (across many blocks)
 522 ;;   vadd (implicit use a7)  ====> emit: vsetvl a7,zero
 523 ;; Such cases are all safe for VSETVL PASS to emit a vsetvl
 524 ;; instruction that modifies the AVL operand.
 525 (define_insn "@vlmax_avl<mode>"
 526   [(set (match_operand:P 0 "register_operand" "=r")
 527         (unspec:P [(match_operand:P 1 "const_int_operand" "i")] UNSPEC_VLMAX))]
 528   "TARGET_VECTOR"
 529   "")
 530
 531 ;; -----------------------------------------------------------------
 532 ;; ---- Moves Operations
 533 ;; -----------------------------------------------------------------
 534
 535 (define_expand "mov<mode>"
 536   [(set (match_operand:V 0 "reg_or_mem_operand")
 537         (match_operand:V 1 "general_operand"))]
 538   "TARGET_VECTOR"
 539 {
 540   /* For whole register move, we transform the pattern into the format
 541      that excludes the clobber of scratch register.
 542
 543      We include clobber of a scalar scratch register which is going to be
 544      used for emit of vsetvl instruction after reload_completed since we
 545      need vsetvl instruction to set VL/VTYPE global status for fractional
 546      vector load/store.
 547
 548      For example:
 549        [(set (match_operand:VNx1QI v24)
 550              (match_operand:VNx1QI (mem: a4)))
 551              (clobber (scratch:SI a5))]
 552      ====>> vsetvl a5,zero,e8,mf8
 553      ====>> vle8.v v24,(a4)
 554
 555      Philosophy:
 556
 557        - Clobber a scalar scratch register for each mov<mode>.
 558
 559        - Classify the machine_mode mode = <MODE>mode into 2 class:
 560          Whole register move and fractional register move.
 561
 562        - Transform and remove scratch clobber register for whole
 563          register move so that we can avoid occupying the scalar
 564          registers.
 565
 566        - We can not leave it to TARGET_SECONDARY_RELOAD since it happens
 567          before spilling. The clobber scratch is used by spilling fractional
 568          registers in IRA/LRA so it's too early.  */
 569
 570   if (riscv_vector::legitimize_move (operands[0], operands[1], <VM>mode))
 571     DONE;
 572 })
 573
 574 ;; This pattern is used for code-gen for whole register load/stores.
 575 ;; Also applicable for all register moves.
 576 ;; Fractional vector modes load/store are not allowed to match this pattern.
 577 ;; Mask modes load/store are not allowed to match this pattern.
 578 ;; We seperate "*mov<mode>" into "*mov<mode>_whole" and "*mov<mode>_fract" because
 579 ;; we don't want to include fractional load/store in "*mov<mode>" which will
 580 ;; create unexpected patterns in LRA.
 581 ;; For example:
 582 ;; ira rtl:
 583 ;;   (insn 20 19 9 2 (set (reg/v:VNx2QI 97 v1 [ v1 ])
 584 ;;      (reg:VNx2QI 134 [ _1 ])) "rvv.c":9:22 571 {*movvnx2qi_fract}
 585 ;;   (nil))
 586 ;; When the value of pseudo register 134 of the insn above is discovered already
 587 ;; spilled in the memory during LRA.
 588 ;; LRA will reload this pattern into a memory load instruction pattern.
 589 ;; Because VNx2QI is a fractional vector, we want LRA reload this pattern into
 590 ;;  (insn 20 19 9 2 (parallel [
 591 ;;       (set (reg:VNx2QI 98 v2 [orig:134 _1 ] [134])
 592 ;;           (mem/c:VNx2QI (reg:SI 13 a3 [155]) [1 %sfp+[-2, -2] S[2, 2] A8]))
 593 ;;       (clobber (reg:SI 14 a4 [149]))])
 594 ;; So that we could be able to emit vsetvl instruction using clobber sratch a4.
 595 ;; To let LRA generate the expected pattern, we should exclude fractional vector
 596 ;; load/store in "*mov<mode>_whole". Otherwise, it will reload this pattern into:
 597 ;;  (insn 20 19 9 2 (set (reg:VNx2QI 98 v2 [orig:134 _1 ] [134])
 598 ;;           (mem/c:VNx2QI (reg:SI 13 a3 [155]) [1 %sfp+[-2, -2] S[2, 2] A8])))
 599 ;; which is not the pattern we want.
 600 ;; According the facts above, we make "*mov<mode>_whole" includes load/store/move for whole
 601 ;; vector modes according to '-march' and "*mov<mode>_fract" only include fractional vector modes.
 602 (define_insn "*mov<mode>_whole"
 603   [(set (match_operand:V_WHOLE 0 "reg_or_mem_operand" "=vr, m,vr")
 604         (match_operand:V_WHOLE 1 "reg_or_mem_operand" "  m,vr,vr"))]
 605   "TARGET_VECTOR"
 606   "@
 607    vl%m1re<sew>.v\t%0,%1
 608    vs%m1r.v\t%1,%0
 609    vmv%m1r.v\t%0,%1"
 610   [(set_attr "type" "vldr,vstr,vmov")
 611    (set_attr "mode" "<MODE>")])
 612
 613 (define_insn "*mov<mode>_fract"
 614   [(set (match_operand:V_FRACT 0 "register_operand" "=vr")
 615         (match_operand:V_FRACT 1 "register_operand" " vr"))]
 616   "TARGET_VECTOR"
 617   "vmv1r.v\t%0,%1"
 618   [(set_attr "type" "vmov")
 619    (set_attr "mode" "<MODE>")])
 620
 621 (define_expand "mov<mode>"
 622   [(set (match_operand:VB 0 "reg_or_mem_operand")
 623         (match_operand:VB 1 "general_operand"))]
 624   "TARGET_VECTOR"
 625 {
 626   if (riscv_vector::legitimize_move (operands[0], operands[1], <MODE>mode))
 627     DONE;
 628 })
 629
 630 (define_insn "*mov<mode>"
 631   [(set (match_operand:VB 0 "register_operand" "=vr")
 632         (match_operand:VB 1 "register_operand" " vr"))]
 633   "TARGET_VECTOR"
 634   "vmv1r.v\t%0,%1"
 635   [(set_attr "type" "vmov")
 636    (set_attr "mode" "<MODE>")])
 637
 638 (define_expand "@mov<V_FRACT:mode><P:mode>_lra"
 639   [(parallel
 640     [(set (match_operand:V_FRACT 0 "reg_or_mem_operand")
 641           (match_operand:V_FRACT 1 "reg_or_mem_operand"))
 642    (clobber (match_scratch:P 2))])]
 643   "TARGET_VECTOR && (lra_in_progress || reload_completed)"
 644 {})
 645
 646 (define_expand "@mov<VB:mode><P:mode>_lra"
 647   [(parallel
 648     [(set (match_operand:VB 0 "reg_or_mem_operand")
 649           (match_operand:VB 1 "reg_or_mem_operand"))
 650    (clobber (match_scratch:P 2))])]
 651   "TARGET_VECTOR && (lra_in_progress || reload_completed)"
 652 {})
 653
 654 (define_insn_and_split "*mov<V_FRACT:mode><P:mode>_lra"
 655   [(set (match_operand:V_FRACT 0 "reg_or_mem_operand" "=vr, m,vr")
 656         (match_operand:V_FRACT 1 "reg_or_mem_operand" "  m,vr,vr"))
 657    (clobber (match_scratch:P 2 "=&r,&r,X"))]
 658   "TARGET_VECTOR && (lra_in_progress || reload_completed)"
 659   "#"
 660   "&& reload_completed"
 661   [(const_int 0)]
 662 {
 663   if (REG_P (operands[0]) && REG_P (operands[1]))
 664       emit_insn (gen_rtx_SET (operands[0], operands[1]));
 665   else
 666     {
 667       riscv_vector::emit_vlmax_vsetvl (<V_FRACT:MODE>mode, operands[2]);
 668       riscv_vector::emit_vlmax_op (code_for_pred_mov (<V_FRACT:MODE>mode),
 669                 operands[0], operands[1], operands[2], <VM>mode);
 670     }
 671   DONE;
 672 })
 673
 674 (define_insn_and_split "*mov<VB:mode><P:mode>_lra"
 675   [(set (match_operand:VB 0 "reg_or_mem_operand" "=vr, m,vr")
 676         (match_operand:VB 1 "reg_or_mem_operand" "  m,vr,vr"))
 677    (clobber (match_scratch:P 2 "=&r,&r,X"))]
 678   "TARGET_VECTOR && (lra_in_progress || reload_completed)"
 679   "#"
 680   "&& reload_completed"
 681   [(const_int 0)]
 682 {
 683   if (REG_P (operands[0]) && REG_P (operands[1]))
 684       emit_insn (gen_rtx_SET (operands[0], operands[1]));
 685   else
 686     {
 687       riscv_vector::emit_vlmax_vsetvl (<VB:MODE>mode, operands[2]);
 688       riscv_vector::emit_vlmax_op (code_for_pred_mov (<VB:MODE>mode),
 689                 operands[0], operands[1], operands[2], <VB:MODE>mode);
 690     }
 691   DONE;
 692 })
 693
 694 ;; -----------------------------------------------------------------
 695 ;; ---- Duplicate Operations
 696 ;; -----------------------------------------------------------------
 697
 698 ;; According to GCC internal:
 699 ;; This pattern only handles duplicates of non-constant inputs.
 700 ;; Constant vectors go through the movm pattern instead.
 701 ;; So "direct_broadcast_operand" can only be mem or reg, no CONSTANT.
 702 (define_expand "vec_duplicate<mode>"
 703   [(set (match_operand:V 0 "register_operand")
 704         (vec_duplicate:V
 705           (match_operand:<VEL> 1 "direct_broadcast_operand")))]
 706   "TARGET_VECTOR"
 707   {
 708     riscv_vector::emit_vlmax_op (code_for_pred_broadcast (<MODE>mode),
 709                 operands[0], operands[1], <VM>mode);
 710     DONE;
 711   }
 712 )
 713
 714 ;; -----------------------------------------------------------------
 715 ;; ---- 6. Configuration-Setting Instructions
 716 ;; -----------------------------------------------------------------
 717 ;; Includes:
 718 ;; - 6.1 vsetvli/vsetivl/vsetvl instructions
 719 ;; -----------------------------------------------------------------
 720
 721 ;; we dont't define vsetvli as unspec_volatile which has side effects.
 722 ;; This instruction can be scheduled by the instruction scheduler.
 723 ;; This means these instructions will be deleted when
 724 ;; there is no instructions using vl or vtype in the following.
 725 ;; rd  | rs1 | AVL value | Effect on vl
 726 ;; -   | !x0 | x[rs1]    | Normal stripmining
 727 ;; !x0 | x0  | ~0        | Set vl to VLMAX
 728 ;; operands[0]: VL.
 729 ;; operands[1]: AVL.
 730 ;; operands[2]: SEW
 731 ;; operands[3]: LMUL
 732 ;; operands[4]: Tail policy 0 or 1 (undisturbed/agnostic)
 733 ;; operands[5]: Mask policy 0 or 1 (undisturbed/agnostic)
 734
 735 ;; We define 2 types of "vsetvl*" instruction patterns:
 736
 737 ;; -  "@vsetvl<mode>" is a parallel format which has side effects.
 738
 739 ;; -  "@vsetvl<mode>_no_side_effects" has no side effects.
 740
 741 ;; -  "@vsetvl<mode>" is used by "vsetvl" intrinsics and "insert-vsetvl" PASS.
 742
 743 ;; -  "@vsetvl<mode>_no_side_effects" is used by GCC standard patterns.
 744
 745 ;; -  "@vsetvl<mode>" includes VL/VTYPE global registers status (define set)
 746 ;; and each RVV instruction includes VL/VTYPE global registers status (use)
 747 ;; so that we can guarantee each RVV instruction can execute with correct
 748 ;; VL/VTYPE global registers status after "insert-vsetvl" PASS.
 749
 750 ;; -  "@vsetvl<mode>_no_side_effects" has no side effects and excludes VL/VTYPE
 751 ;; global registers status (define set). It's only used by GCC standard pattern
 752 ;; expansion. For example: "mov<mode>" pattern for fractional vector modes which
 753 ;; need to set VL/VTYPE. Then we could manually call this pattern to gain benefits
 754 ;; from the optimization of each GCC internal PASS.
 755
 756 ;; 1. void foo (float *in, float *out)
 757 ;;    {
 758 ;;      vfloat32mf2_t v = *(vfloat32mf2_t*)in;
 759 ;;      *(vfloat32mf2_t*)out = v;
 760 ;;    }
 761 ;; We could eliminate the second "vsetvl" by calling "@vsetvl<mode>_no_side_effects".
 762 ;;
 763 ;; "@vsetvl<mode>":               ;; "@vsetvl<mode>_no_side_effects":
 764 ;; vsetvli a4,zero,e32,mf2,ta,ma  ;; vsetvli a4,zero,e32,mf2,ta,ma
 765 ;; vle32.v v24,(a0)               ;; vle32.v v24,(a0)
 766 ;; vsetvli a4,zero,e32,mf2,ta,ma  ;; --
 767 ;; vse32.v v24,(a1)               ;; vse32.v v24,(a1)
 768 ;; ret                            ;; ret
 769
 770 ;; 2. void foo (int8_t *in, int8_t *out, int M)
 771 ;;    {
 772 ;;      for (int i = 0; i < M; i++){
 773 ;;        vint8mf2_t v = *(vint8mf2_t*)(in + i);
 774 ;;        *(vint8mf2_t*)(out + i) = v;
 775 ;;      }
 776 ;;    }
 777 ;;
 778 ;; Hoist "vsetvl" instruction in LICM:
 779 ;; "@vsetvl<mode>":                  ;; "@vsetvl<mode>_no_side_effects":
 780 ;; -                                 ;;   vsetvli a4,zero,e32,mf2,ta,ma
 781 ;; LOOP:                             ;; LOOP:
 782 ;;   vsetvli a4,zero,e32,mf2,ta,ma   ;; -
 783 ;;   vle32.v v24,(a0)                ;;   vle32.v v24,(a0)
 784 ;;   vsetvli a4,zero,e32,mf2,ta,ma   ;; -
 785 ;;   vse32.v v24,(a1)                ;;   vse32.v v24,(a1)
 786
 787 ;; However, it may produce wrong codegen if we exclude VL/VTYPE in "vsevl<mode>".
 788 ;; 3. void foo (int8_t *in, int8_t *out, int32_t *in2, int32_t *out2, int M)
 789 ;;    {
 790 ;;      for (int i = 0; i < M; i++){
 791 ;;        vint8mf2_t v = *(vint8mf2_t*)(in + i);
 792 ;;        vint32mf2_t v2 = *(vint32mf2_t*)(in + i + i);
 793 ;;        *(vint8mf2_t*)(out + i) = v;
 794 ;;        *(vint32mf2_t*)(out + i + i) = v2;
 795 ;;      }
 796 ;;    }
 797 ;;
 798 ;; vsetvli a6,zero,e8,mf2,ta,ma
 799 ;; vsetvli a2,zero,e32,mf2,ta,ma
 800 ;; LOOP:
 801 ;;   vle8.v  v25,(a0)
 802 ;;   vle32.v v24,(a5)
 803 ;;   addi    a0,a0,1
 804 ;;   vse8.v  v25,(a1)
 805 ;;   vse32.v v24,(a3)
 806 ;;
 807 ;; Both vle8.v and vle32.v are using the wrong VL/VTYPE status.
 808 ;; We leave it to "insert-vsetvl" PASS to correct this situation.
 809
 810 ;; The "insert-vsetvl" PASS mechanism:
 811 ;; 1. Before "insert-vsetvl" PASS, only RVV instructions are generated
 812 ;;    by GCC standard pattern expansion has the corresponding "vsetvl".
 813 ;;    We exploit each GCC internal optimization pass to optimize the "vsetvl".
 814 ;; 2. Correct the VL/VTYPE status for each GCC standard pattern RVV instructions.
 815 ;;    Insert vsetvl for each RVV instructions that has no VL/VTYPE status if necessary.
 816 ;;    For example: RVV intrinsics.
 817 ;; 3. Optimize "vsetvl" instructions.
 818
 819 (define_insn "@vsetvl<mode>"
 820   [(set (match_operand:P 0 "register_operand" "=r")
 821         (unspec:P [(match_operand:P 1 "csr_operand" "rK")
 822                    (match_operand 2 "const_int_operand" "i")
 823                    (match_operand 3 "const_int_operand" "i")
 824                    (match_operand 4 "const_int_operand" "i")
 825                    (match_operand 5 "const_int_operand" "i")] UNSPEC_VSETVL))
 826    (set (reg:SI VL_REGNUM)
 827         (unspec:SI [(match_dup 1)
 828                     (match_dup 2)
 829                     (match_dup 3)] UNSPEC_VSETVL))
 830    (set (reg:SI VTYPE_REGNUM)
 831         (unspec:SI [(match_dup 2)
 832                     (match_dup 3)
 833                     (match_dup 4)
 834                     (match_dup 5)] UNSPEC_VSETVL))]
 835   "TARGET_VECTOR"
 836   "vset%i1vli\t%0,%1,e%2,%m3,t%p4,m%p5"
 837   [(set_attr "type" "vsetvl")
 838    (set_attr "mode" "<MODE>")
 839    (set (attr "sew") (symbol_ref "INTVAL (operands[2])"))
 840    (set (attr "vlmul") (symbol_ref "INTVAL (operands[3])"))
 841    (set (attr "ta") (symbol_ref "INTVAL (operands[4])"))
 842    (set (attr "ma") (symbol_ref "INTVAL (operands[5])"))])
 843
 844 ;; vsetvl zero,zero,vtype instruction.
 845 ;; This pattern has no side effects and does not set X0 register.
 846 (define_insn "vsetvl_vtype_change_only"
 847   [(set (reg:SI VTYPE_REGNUM)
 848         (unspec:SI
 849           [(match_operand 0 "const_int_operand" "i")
 850            (match_operand 1 "const_int_operand" "i")
 851            (match_operand 2 "const_int_operand" "i")
 852            (match_operand 3 "const_int_operand" "i")] UNSPEC_VSETVL))]
 853   "TARGET_VECTOR"
 854   "vsetvli\tzero,zero,e%0,%m1,t%p2,m%p3"
 855   [(set_attr "type" "vsetvl")
 856    (set_attr "mode" "SI")])
 857
 858 ;; vsetvl zero,rs1,vtype instruction.
 859 ;; The reason we need this pattern since we should avoid setting X0 register
 860 ;; in vsetvl instruction pattern.
 861 (define_insn "@vsetvl_discard_result<mode>"
 862   [(set (reg:SI VL_REGNUM)
 863         (unspec:SI [(match_operand:P 0 "csr_operand" "rK")
 864                     (match_operand 1 "const_int_operand" "i")
 865                     (match_operand 2 "const_int_operand" "i")] UNSPEC_VSETVL))
 866    (set (reg:SI VTYPE_REGNUM)
 867         (unspec:SI [(match_dup 1)
 868                     (match_dup 2)
 869                     (match_operand 3 "const_int_operand" "i")
 870                     (match_operand 4 "const_int_operand" "i")] UNSPEC_VSETVL))]
 871   "TARGET_VECTOR"
 872   "vset%i0vli\tzero,%0,e%1,%m2,t%p3,m%p4"
 873   [(set_attr "type" "vsetvl")
 874    (set_attr "mode" "<MODE>")
 875    (set (attr "sew") (symbol_ref "INTVAL (operands[1])"))
 876    (set (attr "vlmul") (symbol_ref "INTVAL (operands[2])"))
 877    (set (attr "ta") (symbol_ref "INTVAL (operands[3])"))
 878    (set (attr "ma") (symbol_ref "INTVAL (operands[4])"))])
 879
 880 ;; It's emit by vsetvl/vsetvlmax intrinsics with no side effects.
 881 ;; Since we have many optmization passes from "expand" to "reload_completed",
 882 ;; such pattern can allow us gain benefits of these optimizations.
 883 (define_insn_and_split "@vsetvl<mode>_no_side_effects"
 884   [(set (match_operand:P 0 "register_operand" "=r")
 885         (unspec:P [(match_operand:P 1 "csr_operand" "rK")
 886                    (match_operand 2 "const_int_operand" "i")
 887                    (match_operand 3 "const_int_operand" "i")
 888                    (match_operand 4 "const_int_operand" "i")
 889                    (match_operand 5 "const_int_operand" "i")] UNSPEC_VSETVL))]
 890   "TARGET_VECTOR"
 891   "#"
 892   "&& epilogue_completed"
 893   [(parallel
 894     [(set (match_dup 0)
 895           (unspec:P [(match_dup 1) (match_dup 2) (match_dup 3)
 896                      (match_dup 4) (match_dup 5)] UNSPEC_VSETVL))
 897      (set (reg:SI VL_REGNUM)
 898           (unspec:SI [(match_dup 1) (match_dup 2) (match_dup 3)] UNSPEC_VSETVL))
 899      (set (reg:SI VTYPE_REGNUM)
 900           (unspec:SI [(match_dup 2) (match_dup 3) (match_dup 4)
 901                       (match_dup 5)] UNSPEC_VSETVL))])]
 902   ""
 903   [(set_attr "type" "vsetvl")
 904    (set_attr "mode" "SI")])
 905
 906 ;; RVV machine description matching format
 907 ;; (define_insn ""
 908 ;;   [(set (match_operand:MODE 0)
 909 ;;      (if_then_else:MODE
 910 ;;        (unspec:<MODE:VM>
 911 ;;          [(match_operand:<VM> 1 "vector_mask_operand")
 912 ;;           (match_operand N + 4 "vector_length_operand")
 913 ;;           (match_operand N + 5 "const_int_operand")
 914 ;;           (match_operand N + 6 "const_int_operand")
 915 ;;           (reg:SI VL_REGNUM)
 916 ;;           (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
 917 ;;        (instruction operation:MODE
 918 ;;           (match_operand 3
 919 ;;           (match_operand 4
 920 ;;           (match_operand 5
 921 ;;           ................
 922 ;;           (match_operand N + 3)
 923 ;;        (match_operand:MODE 2 "vector_reg_or_const0_operand")))]
 924 ;;
 925 ;; (unspec:[........] UNSPEC_VPREDICATE) is a predicate wrapper.
 926 ;; Include mask predicate && length predicate && vector policy.
 927
 928 ;; -------------------------------------------------------------------------------
 929 ;; ---- Predicated Mov
 930 ;; -------------------------------------------------------------------------------
 931 ;; Includes:
 932 ;; - 7.4. Vector Unit-Stride Instructions
 933 ;; - 11.15 Vector Integer Merge Instructions
 934 ;; - 11.16 Vector Integer Move Instructions
 935 ;; - 13.16 Vector Floating-Point Move Instruction
 936 ;; - 15.1 Vector Mask-Register Logical Instructions
 937 ;; -------------------------------------------------------------------------------
 938
 939 ;; vle.v/vse.v/vmv.v.v/vmv.v.x/vmv.v.i/vfmv.v.f.
 940 ;; For vle.v/vmv.v.v/vmv.v.x/vmv.v.i/vfmv.v.f, we may need merge and mask operand.
 941 ;; For vse.v, we don't need merge operand, so it should always match "vu".
 942 ;; constraint alternative 0 ~ 1 match vle.v.
 943 ;; constraint alternative 2 match vse.v.
 944 ;; constraint alternative 3 match vmv.v.v.
 945 ;; constraint alternative 4 match vmv.v.i.
 946 ;; For vmv.v.i, we allow 2 following cases:
 947 ;;    1. (const_vector:VNx1QI repeat [
 948 ;;                (const_int:QI N)]), -15 <= N < 16.
 949 ;;    2. (const_vector:VNx1SF repeat [
 950 ;;                (const_double:SF 0.0 [0x0.0p+0])]).
 951
 952 ;; We add "MEM_P (operands[0]) || MEM_P (operands[3]) || CONST_VECTOR_P (operands[1])" here to
 953 ;; make sure we don't want CSE to generate the following pattern:
 954 ;; (insn 17 8 19 2 (set (reg:VNx1HI 134 [ _1 ])
 955 ;;       (if_then_else:VNx1HI (unspec:VNx1BI [
 956 ;;                   (reg/v:VNx1BI 137 [ mask ])
 957 ;;                   (reg:DI 151)
 958 ;;                   (const_int 0 [0]) repeated x3
 959 ;;                   (reg:SI 66 vl)
 960 ;;                   (reg:SI 67 vtype)
 961 ;;               ] UNSPEC_VPREDICATE)
 962 ;;           (const_vector:VNx1HI repeat [
 963 ;;                   (const_int 0 [0])
 964 ;;               ])
 965 ;;           (reg/v:VNx1HI 140 [ merge ]))) "rvv.c":8:12 608 {pred_movvnx1hi}
 966 ;;    (expr_list:REG_DEAD (reg:DI 151)
 967 ;;       (expr_list:REG_DEAD (reg/v:VNx1HI 140 [ merge ])
 968 ;;           (expr_list:REG_DEAD (reg/v:VNx1BI 137 [ mask ])
 969 ;;               (nil)))))
 970 ;; Since both vmv.v.v and vmv.v.i doesn't have mask operand.
 971 (define_insn_and_split "@pred_mov<mode>"
 972   [(set (match_operand:V 0 "nonimmediate_operand"      "=vr,    vr,    vd,     m,    vr,    vr,    vr,    vr")
 973     (if_then_else:V
 974       (unspec:<VM>
 975         [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,   Wc1,    vm, vmWc1,   Wc1,   Wc1,   Wc1,   Wc1")
 976          (match_operand 4 "vector_length_operand"    "   rK,    rK,    rK,    rK,    rK,    rK,    rK,    rK")
 977          (match_operand 5 "const_int_operand"        "    i,     i,     i,     i,     i,     i,     i,     i")
 978          (match_operand 6 "const_int_operand"        "    i,     i,     i,     i,     i,     i,     i,     i")
 979          (match_operand 7 "const_int_operand"        "    i,     i,     i,     i,     i,     i,     i,     i")
 980          (reg:SI VL_REGNUM)
 981          (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
 982       (match_operand:V 3 "vector_move_operand"       "    m,     m,     m,    vr,    vr,    vr, viWc0, viWc0")
 983       (match_operand:V 2 "vector_merge_operand"      "    0,    vu,    vu,    vu,    vu,     0,    vu,     0")))]
 984   "TARGET_VECTOR && (MEM_P (operands[0]) || MEM_P (operands[3])
 985    || CONST_VECTOR_P (operands[1]))"
 986   "@
 987    vle<sew>.v\t%0,%3%p1
 988    vle<sew>.v\t%0,%3
 989    vle<sew>.v\t%0,%3,%1.t
 990    vse<sew>.v\t%3,%0%p1
 991    vmv.v.v\t%0,%3
 992    vmv.v.v\t%0,%3
 993    vmv.v.i\t%0,%v3
 994    vmv.v.i\t%0,%v3"
 995   "&& register_operand (operands[0], <MODE>mode)
 996    && register_operand (operands[3], <MODE>mode)
 997    && satisfies_constraint_vu (operands[2])
 998    && INTVAL (operands[7]) == riscv_vector::VLMAX"
 999   [(set (match_dup 0) (match_dup 3))]
1000   ""
1001   [(set_attr "type" "vlde,vlde,vlde,vste,vimov,vimov,vimov,vimov")
1002    (set_attr "mode" "<MODE>")])
1003
1004 ;; Dedicated pattern for vse.v instruction since we can't reuse pred_mov pattern to include
1005 ;; memory operand as input which will produce inferior codegen.
1006 (define_insn "@pred_store<mode>"
1007   [(set (match_operand:V 0 "memory_operand"                 "+m")
1008         (if_then_else:V
1009           (unspec:<VM>
1010             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
1011              (match_operand 3 "vector_length_operand"    "   rK")
1012              (reg:SI VL_REGNUM)
1013              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1014           (match_operand:V 2 "register_operand"         "    vr")
1015           (match_dup 0)))]
1016   "TARGET_VECTOR"
1017   "vse<sew>.v\t%2,%0%p1"
1018   [(set_attr "type" "vste")
1019    (set_attr "mode" "<MODE>")
1020    (set (attr "avl_type") (symbol_ref "riscv_vector::NONVLMAX"))
1021    (set_attr "vl_op_idx" "3")])
1022
1023 ;; vlm.v/vsm.v/vmclr.m/vmset.m.
1024 ;; constraint alternative 0 match vlm.v.
1025 ;; constraint alternative 1 match vsm.v.
1026 ;; constraint alternative 3 match vmclr.m.
1027 ;; constraint alternative 4 match vmset.m.
1028 (define_insn_and_split "@pred_mov<mode>"
1029   [(set (match_operand:VB 0 "nonimmediate_operand"               "=vr,   m,  vr,  vr,  vr")
1030         (if_then_else:VB
1031           (unspec:VB
1032             [(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1, Wc1, Wc1, Wc1, Wc1")
1033              (match_operand 4 "vector_length_operand"            " rK,  rK,  rK,  rK,  rK")
1034              (match_operand 5 "const_int_operand"                "  i,   i,   i,   i,   i")
1035              (reg:SI VL_REGNUM)
1036              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1037           (match_operand:VB 3 "vector_move_operand"              "  m,  vr,  vr, Wc0, Wc1")
1038           (match_operand:VB 2 "vector_undef_operand"             " vu,  vu,  vu,  vu,  vu")))]
1039   "TARGET_VECTOR"
1040   "@
1041    vlm.v\t%0,%3
1042    vsm.v\t%3,%0
1043    vmmv.m\t%0,%3
1044    vmclr.m\t%0
1045    vmset.m\t%0"
1046   "&& register_operand (operands[0], <MODE>mode)
1047    && register_operand (operands[3], <MODE>mode)
1048    && INTVAL (operands[5]) == riscv_vector::VLMAX"
1049   [(set (match_dup 0) (match_dup 3))]
1050   ""
1051   [(set_attr "type" "vldm,vstm,vmalu,vmalu,vmalu")
1052    (set_attr "mode" "<MODE>")])
1053
1054 ;; Dedicated pattern for vsm.v instruction since we can't reuse pred_mov pattern to include
1055 ;; memory operand as input which will produce inferior codegen.
1056 (define_insn "@pred_store<mode>"
1057   [(set (match_operand:VB 0 "memory_operand"                      "+m")
1058         (if_then_else:VB
1059           (unspec:VB
1060             [(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1")
1061              (match_operand 3 "vector_length_operand"            " rK")
1062              (reg:SI VL_REGNUM)
1063              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1064           (match_operand:VB 2 "register_operand"                 " vr")
1065           (match_dup 0)))]
1066   "TARGET_VECTOR"
1067   "vsm.v\t%2,%0"
1068   [(set_attr "type" "vstm")
1069    (set_attr "mode" "<MODE>")
1070    (set (attr "avl_type") (symbol_ref "riscv_vector::NONVLMAX"))
1071    (set_attr "vl_op_idx" "3")])
1072
1073 (define_insn "@pred_merge<mode>"
1074   [(set (match_operand:V 0 "register_operand"        "=vd,vd,vd,vd")
1075     (if_then_else:V
1076       (unspec:<VM>
1077         [(match_operand 5 "vector_length_operand"    " rK,rK,rK,rK")
1078          (match_operand 6 "const_int_operand"        "  i, i, i, i")
1079          (match_operand 7 "const_int_operand"        "  i, i, i, i")
1080          (reg:SI VL_REGNUM)
1081          (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1082       (vec_merge:V
1083         (match_operand:V 3 "vector_arith_operand"    " vr,vr,vi,vi")
1084         (match_operand:V 2 "register_operand"        " vr,vr,vr,vr")
1085         (match_operand:<VM> 4 "register_operand"     " vm,vm,vm,vm"))
1086       (match_operand:V 1 "vector_merge_operand"      " vu, 0,vu, 0")))]
1087   "TARGET_VECTOR"
1088   "vmerge.v%o3m\t%0,%2,%v3,%4"
1089   [(set_attr "type" "vimerge")
1090    (set_attr "mode" "<MODE>")])
1091
1092 (define_insn "@pred_merge<mode>_scalar"
1093   [(set (match_operand:VI_QHS 0 "register_operand"   "=vd,vd")
1094     (if_then_else:VI_QHS
1095       (unspec:<VM>
1096         [(match_operand 5 "vector_length_operand"    " rK,rK")
1097          (match_operand 6 "const_int_operand"        "  i, i")
1098          (match_operand 7 "const_int_operand"        "  i, i")
1099          (reg:SI VL_REGNUM)
1100          (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1101       (vec_merge:VI_QHS
1102         (vec_duplicate:VI_QHS
1103           (match_operand:<VEL> 3 "register_operand"  "  r, r"))
1104         (match_operand:VI_QHS 2 "register_operand"   " vr,vr")
1105         (match_operand:<VM> 4 "register_operand"     " vm,vm"))
1106       (match_operand:VI_QHS 1 "vector_merge_operand" " vu, 0")))]
1107   "TARGET_VECTOR"
1108   "vmerge.vxm\t%0,%2,%3,%4"
1109   [(set_attr "type" "vimerge")
1110    (set_attr "mode" "<MODE>")])
1111
1112 (define_expand "@pred_merge<mode>_scalar"
1113   [(set (match_operand:VI_D 0 "register_operand")
1114     (if_then_else:VI_D
1115       (unspec:<VM>
1116         [(match_operand 5 "vector_length_operand")
1117          (match_operand 6 "const_int_operand")
1118          (match_operand 7 "const_int_operand")
1119          (reg:SI VL_REGNUM)
1120          (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1121       (vec_merge:VI_D
1122         (vec_duplicate:VI_D
1123           (match_operand:<VEL> 3 "reg_or_int_operand"))
1124         (match_operand:VI_D 2 "register_operand")
1125         (match_operand:<VM> 4 "register_operand"))
1126       (match_operand:VI_D 1 "vector_merge_operand")))]
1127   "TARGET_VECTOR"
1128 {
1129   if (riscv_vector::sew64_scalar_helper (
1130         operands,
1131         /* scalar op */&operands[3],
1132         /* vl */operands[5],
1133         <MODE>mode,
1134         <VM>mode,
1135         riscv_vector::simm5_p (operands[3]),
1136         [] (rtx *operands, rtx boardcast_scalar) {
1137           emit_insn (gen_pred_merge<mode> (operands[0], operands[1],
1138                operands[2], boardcast_scalar, operands[4], operands[5],
1139                operands[6], operands[7]));
1140         }))
1141     DONE;
1142 })
1143
1144 (define_insn "*pred_merge<mode>_scalar"
1145   [(set (match_operand:VI_D 0 "register_operand"     "=vd,vd")
1146     (if_then_else:VI_D
1147       (unspec:<VM>
1148         [(match_operand 5 "vector_length_operand"    " rK,rK")
1149          (match_operand 6 "const_int_operand"        "  i, i")
1150          (match_operand 7 "const_int_operand"        "  i, i")
1151          (reg:SI VL_REGNUM)
1152          (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1153       (vec_merge:VI_D
1154         (vec_duplicate:VI_D
1155           (match_operand:<VEL> 3 "register_operand"  "  r, r"))
1156         (match_operand:VI_D 2 "register_operand"     " vr,vr")
1157         (match_operand:<VM> 4 "register_operand"     " vm,vm"))
1158       (match_operand:VI_D 1 "vector_merge_operand"   " vu, 0")))]
1159   "TARGET_VECTOR"
1160   "vmerge.vxm\t%0,%2,%3,%4"
1161   [(set_attr "type" "vimerge")
1162    (set_attr "mode" "<MODE>")])
1163
1164 (define_insn "*pred_merge<mode>_extended_scalar"
1165   [(set (match_operand:VI_D 0 "register_operand"         "=vd,vd")
1166     (if_then_else:VI_D
1167       (unspec:<VM>
1168         [(match_operand 5 "vector_length_operand"        " rK,rK")
1169          (match_operand 6 "const_int_operand"            "  i, i")
1170          (match_operand 7 "const_int_operand"            "  i, i")
1171          (reg:SI VL_REGNUM)
1172          (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1173       (vec_merge:VI_D
1174         (vec_duplicate:VI_D
1175           (sign_extend:<VEL>
1176             (match_operand:<VSUBEL> 3 "register_operand" "  r, r")))
1177         (match_operand:VI_D 2 "register_operand"         " vr,vr")
1178         (match_operand:<VM> 4 "register_operand"         " vm,vm"))
1179       (match_operand:VI_D 1 "vector_merge_operand"       " vu, 0")))]
1180   "TARGET_VECTOR"
1181   "vmerge.vxm\t%0,%2,%3,%4"
1182   [(set_attr "type" "vimerge")
1183    (set_attr "mode" "<MODE>")])
1184
1185 ;; -------------------------------------------------------------------------------
1186 ;; ---- Predicated Broadcast
1187 ;; -------------------------------------------------------------------------------
1188 ;; Includes:
1189 ;; - 7.5. Vector Strided Instructions (zero stride)
1190 ;; - 11.16 Vector Integer Move Instructions (vmv.v.x)
1191 ;; - 13.16 Vector Floating-Point Move Instruction (vfmv.v.f)
1192 ;; - 16.1 Integer Scalar Move Instructions (vmv.s.x)
1193 ;; - 16.2 Floating-Point Scalar Move Instructions (vfmv.s.f)
1194 ;; -------------------------------------------------------------------------------
1195
1196 ;; According to RVV ISA, vector-scalar instruction doesn't support
1197 ;; operand fetched from 2 consecutive registers, so we should use
1198 ;; vlse.v which is a memory access to broadcast a DImode scalar into a vector.
1199 ;;
1200 ;; Since the optimization flow in GCC is as follows:
1201 ;; expand --> LICM (Loop invariant) --> split.
1202 ;; To use LICM optimization, we postpone generation of vlse.v to split stage since
1203 ;; a memory access instruction can not be optimized by LICM (Loop invariant).
1204 (define_expand "@pred_broadcast<mode>"
1205   [(set (match_operand:V 0 "register_operand")
1206         (if_then_else:V
1207           (unspec:<VM>
1208             [(match_operand:<VM> 1 "vector_broadcast_mask_operand")
1209              (match_operand 4 "vector_length_operand")
1210              (match_operand 5 "const_int_operand")
1211              (match_operand 6 "const_int_operand")
1212              (match_operand 7 "const_int_operand")
1213              (reg:SI VL_REGNUM)
1214              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1215           (vec_duplicate:V
1216             (match_operand:<VEL> 3 "direct_broadcast_operand"))
1217           (match_operand:V 2 "vector_merge_operand")))]
1218   "TARGET_VECTOR"
1219 {
1220   /* Handle vmv.s.x instruction which has memory scalar.  */
1221   if (satisfies_constraint_Wdm (operands[3]) || riscv_vector::simm5_p (operands[3])
1222       || rtx_equal_p (operands[3], CONST0_RTX (<VEL>mode)))
1223     {
1224       if (satisfies_constraint_Wb1 (operands[1]))
1225         {
1226           // Case 1: vmv.s.x (TA) ==> vlse.v (TA)
1227           if (satisfies_constraint_vu (operands[2]))
1228             operands[1] = CONSTM1_RTX (<VM>mode);
1229           else if (GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode))
1230             {
1231               // Case 2: vmv.s.x (TU) ==> andi vl + vlse.v (TU) in RV32 system.
1232               operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
1233               operands[1] = CONSTM1_RTX (<VM>mode);
1234             }
1235           else
1236             operands[3] = force_reg (<VEL>mode, operands[3]);
1237         }
1238     }
1239   else if (GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode)
1240            && immediate_operand (operands[3], Pmode))
1241     operands[3] = gen_rtx_SIGN_EXTEND (<VEL>mode, force_reg (Pmode, operands[3]));
1242   else
1243     operands[3] = force_reg (<VEL>mode, operands[3]);
1244 })
1245
1246 (define_insn_and_split "*pred_broadcast<mode>"
1247   [(set (match_operand:VI 0 "register_operand"                     "=vr, vr, vd, vd, vr, vr, vr, vr")
1248         (if_then_else:VI
1249           (unspec:<VM>
1250             [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1, vm, vm,Wc1,Wc1,Wb1,Wb1")
1251              (match_operand 4 "vector_length_operand"              " rK, rK, rK, rK, rK, rK, rK, rK")
1252              (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
1253              (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
1254              (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
1255              (reg:SI VL_REGNUM)
1256              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1257           (vec_duplicate:VI
1258             (match_operand:<VEL> 3 "direct_broadcast_operand"       " r,  r,Wdm,Wdm,Wdm,Wdm,  r,  r"))
1259           (match_operand:VI 2 "vector_merge_operand"                "vu,  0, vu,  0, vu,  0, vu,  0")))]
1260   "TARGET_VECTOR"
1261   "@
1262    vmv.v.x\t%0,%3
1263    vmv.v.x\t%0,%3
1264    vlse<sew>.v\t%0,%3,zero,%1.t
1265    vlse<sew>.v\t%0,%3,zero,%1.t
1266    vlse<sew>.v\t%0,%3,zero
1267    vlse<sew>.v\t%0,%3,zero
1268    vmv.s.x\t%0,%3
1269    vmv.s.x\t%0,%3"
1270   "register_operand (operands[3], <VEL>mode)
1271   && GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode)"
1272   [(set (match_dup 0)
1273         (if_then_else:VI (unspec:<VM> [(match_dup 1) (match_dup 4)
1274              (match_dup 5) (match_dup 6) (match_dup 7)
1275              (reg:SI VL_REGNUM) (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1276           (vec_duplicate:VI (match_dup 3))
1277           (match_dup 2)))]
1278   {
1279     gcc_assert (can_create_pseudo_p ());
1280     rtx m = assign_stack_local (<VEL>mode, GET_MODE_SIZE (<VEL>mode),
1281                                 GET_MODE_ALIGNMENT (<VEL>mode));
1282     m = validize_mem (m);
1283     emit_move_insn (m, operands[3]);
1284     m = gen_rtx_MEM (<VEL>mode, force_reg (Pmode, XEXP (m, 0)));
1285     operands[3] = m;
1286
1287     /* For SEW = 64 in RV32 system, we expand vmv.s.x:
1288        andi a2,a2,1
1289        vsetvl zero,a2,e64
1290        vlse64.v  */
1291     if (satisfies_constraint_Wb1 (operands[1]))
1292       {
1293         operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
1294         operands[1] = CONSTM1_RTX (<VM>mode);
1295       }
1296   }
1297   [(set_attr "type" "vimov,vimov,vlds,vlds,vlds,vlds,vimovxv,vimovxv")
1298    (set_attr "mode" "<MODE>")])
1299
1300 (define_insn "*pred_broadcast<mode>"
1301   [(set (match_operand:VF 0 "register_operand"                     "=vr, vr, vr, vr, vr, vr, vr, vr")
1302         (if_then_else:VF
1303           (unspec:<VM>
1304             [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1, vm, vm,Wc1,Wc1,Wb1,Wb1")
1305              (match_operand 4 "vector_length_operand"              " rK, rK, rK, rK, rK, rK, rK, rK")
1306              (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
1307              (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
1308              (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
1309              (reg:SI VL_REGNUM)
1310              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1311           (vec_duplicate:VF
1312             (match_operand:<VEL> 3 "direct_broadcast_operand"       " f,  f,Wdm,Wdm,Wdm,Wdm,  f,  f"))
1313           (match_operand:VF 2 "vector_merge_operand"                "vu,  0, vu,  0, vu,  0, vu,  0")))]
1314   "TARGET_VECTOR"
1315   "@
1316    vfmv.v.f\t%0,%3
1317    vfmv.v.f\t%0,%3
1318    vlse<sew>.v\t%0,%3,zero,%1.t
1319    vlse<sew>.v\t%0,%3,zero,%1.t
1320    vlse<sew>.v\t%0,%3,zero
1321    vlse<sew>.v\t%0,%3,zero
1322    vfmv.s.f\t%0,%3
1323    vfmv.s.f\t%0,%3"
1324   [(set_attr "type" "vfmov,vfmov,vlds,vlds,vlds,vlds,vfmovfv,vfmovfv")
1325    (set_attr "mode" "<MODE>")])
1326
1327 (define_insn "*pred_broadcast<mode>_extended_scalar"
1328   [(set (match_operand:VI_D 0 "register_operand"                   "=vr, vr, vr, vr")
1329         (if_then_else:VI_D
1330           (unspec:<VM>
1331             [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1,Wb1,Wb1")
1332              (match_operand 4 "vector_length_operand"              " rK, rK, rK, rK")
1333              (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i")
1334              (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i")
1335              (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i")
1336              (reg:SI VL_REGNUM)
1337              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1338           (vec_duplicate:VI_D
1339             (sign_extend:<VEL>
1340               (match_operand:<VSUBEL> 3 "register_operand"          " r,  r,  r,  r")))
1341           (match_operand:VI_D 2 "vector_merge_operand"              "vu,  0, vu,  0")))]
1342   "TARGET_VECTOR"
1343   "@
1344    vmv.v.x\t%0,%3
1345    vmv.v.x\t%0,%3
1346    vmv.s.x\t%0,%3
1347    vmv.s.x\t%0,%3"
1348   [(set_attr "type" "vimov,vimov,vimovxv,vimovxv")
1349    (set_attr "mode" "<MODE>")])
1350
1351 ;; -------------------------------------------------------------------------------
1352 ;; ---- Predicated Strided loads/stores
1353 ;; -------------------------------------------------------------------------------
1354 ;; Includes:
1355 ;; - 7.5. Vector Strided Instructions
1356 ;; -------------------------------------------------------------------------------
1357
1358 (define_insn "@pred_strided_load<mode>"
1359   [(set (match_operand:V 0 "register_operand"              "=vr,    vr,    vd")
1360         (if_then_else:V
1361           (unspec:<VM>
1362             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,   Wc1,    vm")
1363              (match_operand 5 "vector_length_operand"    "   rK,    rK,    rK")
1364              (match_operand 6 "const_int_operand"        "    i,     i,     i")
1365              (match_operand 7 "const_int_operand"        "    i,     i,     i")
1366              (reg:SI VL_REGNUM)
1367              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1368           (unspec:V
1369             [(match_operand:V 3 "memory_operand"         "    m,     m,     m")
1370              (match_operand 4 "pmode_reg_or_0_operand"   "   rJ,    rJ,    rJ")] UNSPEC_STRIDED)
1371           (match_operand:V 2 "vector_merge_operand"      "    0,    vu,    vu")))]
1372   "TARGET_VECTOR"
1373   "vlse<sew>.v\t%0,%3,%z4%p1"
1374   [(set_attr "type" "vlds")
1375    (set_attr "mode" "<MODE>")])
1376
1377 (define_insn "@pred_strided_store<mode>"
1378   [(set (match_operand:V 0 "memory_operand"                 "+m")
1379         (if_then_else:V
1380           (unspec:<VM>
1381             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1")
1382              (match_operand 4 "vector_length_operand"    "   rK")
1383              (reg:SI VL_REGNUM)
1384              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1385           (unspec:V
1386             [(match_operand 2 "pmode_reg_or_0_operand"   "   rJ")
1387              (match_operand:V 3 "register_operand"       "   vr")] UNSPEC_STRIDED)
1388           (match_dup 0)))]
1389   "TARGET_VECTOR"
1390   "vsse<sew>.v\t%3,%0,%z2%p1"
1391   [(set_attr "type" "vsts")
1392    (set_attr "mode" "<MODE>")])
1393
1394 ;; -------------------------------------------------------------------------------
1395 ;; ---- Predicated indexed loads/stores
1396 ;; -------------------------------------------------------------------------------
1397 ;; Includes:
1398 ;; - 7.6. Vector Indexed Instructions
1399 ;; -------------------------------------------------------------------------------
1400
1401 ;; DEST eew is same as SOURCE eew, DEST register can overlap SOURCE.
1402 (define_insn "@pred_indexed_<order>load<mode>_same_eew"
1403   [(set (match_operand:V 0 "register_operand"             "=vd, vr,vd, vr")
1404         (if_then_else:V
1405           (unspec:<VM>
1406             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,Wc1,vm,Wc1")
1407              (match_operand 5 "vector_length_operand"     " rK, rK,rK, rK")
1408              (match_operand 6 "const_int_operand"         "  i,  i, i,  i")
1409              (match_operand 7 "const_int_operand"         "  i,  i, i,  i")
1410              (match_operand 8 "const_int_operand"         "  i,  i, i,  i")
1411              (reg:SI VL_REGNUM)
1412              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1413           (unspec:V
1414             [(match_operand 3 "pmode_register_operand"    "  r,  r, r,  r")
1415              (mem:BLK (scratch))
1416              (match_operand:<VINDEX> 4 "register_operand" " vr, vr,vr, vr")] ORDER)
1417           (match_operand:V 2 "vector_merge_operand"       " vu, vu, 0,  0")))]
1418   "TARGET_VECTOR"
1419   "vl<order>xei<sew>.v\t%0,(%3),%4%p1"
1420   [(set_attr "type" "vld<order>x")
1421    (set_attr "mode" "<MODE>")])
1422
1423 ;; DEST eew is greater than SOURCE eew.
1424 (define_insn "@pred_indexed_<order>load<mode>_x2_greater_eew"
1425   [(set (match_operand:VEEWEXT2 0 "register_operand"                    "=&vr,  &vr")
1426         (if_then_else:VEEWEXT2
1427           (unspec:<VM>
1428             [(match_operand:<VM> 1 "vector_mask_operand"               "vmWc1,vmWc1")
1429              (match_operand 5 "vector_length_operand"                  "   rK,   rK")
1430              (match_operand 6 "const_int_operand"                      "    i,    i")
1431              (match_operand 7 "const_int_operand"                      "    i,    i")
1432              (match_operand 8 "const_int_operand"                      "    i,    i")
1433              (reg:SI VL_REGNUM)
1434              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1435           (unspec:VEEWEXT2
1436             [(match_operand 3 "pmode_register_operand"                 "    r,    r")
1437              (mem:BLK (scratch))
1438              (match_operand:<VINDEX_DOUBLE_TRUNC> 4 "register_operand" "   vr,   vr")] ORDER)
1439           (match_operand:VEEWEXT2 2 "vector_merge_operand"             "   vu,    0")))]
1440   "TARGET_VECTOR"
1441   "vl<order>xei<double_trunc_sew>.v\t%0,(%3),%4%p1"
1442   [(set_attr "type" "vld<order>x")
1443    (set_attr "mode" "<MODE>")])
1444
1445 (define_insn "@pred_indexed_<order>load<mode>_x4_greater_eew"
1446   [(set (match_operand:VEEWEXT4 0 "register_operand"                    "=&vr,  &vr")
1447         (if_then_else:VEEWEXT4
1448           (unspec:<VM>
1449             [(match_operand:<VM> 1 "vector_mask_operand"               "vmWc1,vmWc1")
1450              (match_operand 5 "vector_length_operand"                  "   rK,   rK")
1451              (match_operand 6 "const_int_operand"                      "    i,    i")
1452              (match_operand 7 "const_int_operand"                      "    i,    i")
1453              (match_operand 8 "const_int_operand"                      "    i,    i")
1454              (reg:SI VL_REGNUM)
1455              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1456           (unspec:VEEWEXT4
1457             [(match_operand 3 "pmode_register_operand"                 "    r,    r")
1458              (mem:BLK (scratch))
1459              (match_operand:<VINDEX_QUAD_TRUNC> 4 "register_operand"   "   vr,   vr")] ORDER)
1460           (match_operand:VEEWEXT4 2 "vector_merge_operand"             "   vu,    0")))]
1461   "TARGET_VECTOR"
1462   "vl<order>xei<quad_trunc_sew>.v\t%0,(%3),%4%p1"
1463   [(set_attr "type" "vld<order>x")
1464    (set_attr "mode" "<MODE>")])
1465
1466 (define_insn "@pred_indexed_<order>load<mode>_x8_greater_eew"
1467   [(set (match_operand:VEEWEXT8 0 "register_operand"                    "=&vr,  &vr")
1468         (if_then_else:VEEWEXT8
1469           (unspec:<VM>
1470             [(match_operand:<VM> 1 "vector_mask_operand"               "vmWc1,vmWc1")
1471              (match_operand 5 "vector_length_operand"                  "   rK,   rK")
1472              (match_operand 6 "const_int_operand"                      "    i,    i")
1473              (match_operand 7 "const_int_operand"                      "    i,    i")
1474              (match_operand 8 "const_int_operand"                      "    i,    i")
1475              (reg:SI VL_REGNUM)
1476              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1477           (unspec:VEEWEXT8
1478             [(match_operand 3 "pmode_register_operand"                 "    r,    r")
1479              (mem:BLK (scratch))
1480              (match_operand:<VINDEX_OCT_TRUNC> 4 "register_operand"    "   vr,   vr")] ORDER)
1481           (match_operand:VEEWEXT8 2 "vector_merge_operand"             "   vu,    0")))]
1482   "TARGET_VECTOR"
1483   "vl<order>xei<oct_trunc_sew>.v\t%0,(%3),%4%p1"
1484   [(set_attr "type" "vld<order>x")
1485    (set_attr "mode" "<MODE>")])
1486
1487 ;; DEST eew is smaller than SOURCE eew.
1488 (define_insn "@pred_indexed_<order>load<mode>_x2_smaller_eew"
1489   [(set (match_operand:VEEWTRUNC2 0 "register_operand"                "=&vr,  &vr")
1490         (if_then_else:VEEWTRUNC2
1491           (unspec:<VM>
1492             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1,vmWc1")
1493              (match_operand 5 "vector_length_operand"                "   rK,   rK")
1494              (match_operand 6 "const_int_operand"                    "    i,    i")
1495              (match_operand 7 "const_int_operand"                    "    i,    i")
1496              (match_operand 8 "const_int_operand"                    "    i,    i")
1497              (reg:SI VL_REGNUM)
1498              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1499           (unspec:VEEWTRUNC2
1500             [(match_operand 3 "pmode_register_operand"               "    r,    r")
1501              (mem:BLK (scratch))
1502              (match_operand:<VINDEX_DOUBLE_EXT> 4 "register_operand" "   vr,   vr")] ORDER)
1503           (match_operand:VEEWTRUNC2 2 "vector_merge_operand"         "   vu,    0")))]
1504   "TARGET_VECTOR"
1505   "vl<order>xei<double_ext_sew>.v\t%0,(%3),%4%p1"
1506   [(set_attr "type" "vld<order>x")
1507    (set_attr "mode" "<MODE>")])
1508
1509 (define_insn "@pred_indexed_<order>load<mode>_x4_smaller_eew"
1510   [(set (match_operand:VEEWTRUNC4 0 "register_operand"              "=&vr,  &vr")
1511         (if_then_else:VEEWTRUNC4
1512           (unspec:<VM>
1513             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
1514              (match_operand 5 "vector_length_operand"              "   rK,   rK")
1515              (match_operand 6 "const_int_operand"                  "    i,    i")
1516              (match_operand 7 "const_int_operand"                  "    i,    i")
1517              (match_operand 8 "const_int_operand"                  "    i,    i")
1518              (reg:SI VL_REGNUM)
1519              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1520           (unspec:VEEWTRUNC4
1521             [(match_operand 3 "pmode_register_operand"             "    r,    r")
1522              (mem:BLK (scratch))
1523              (match_operand:<VINDEX_QUAD_EXT> 4 "register_operand" "   vr,   vr")] ORDER)
1524           (match_operand:VEEWTRUNC4 2 "vector_merge_operand"       "   vu,    0")))]
1525   "TARGET_VECTOR"
1526   "vl<order>xei<quad_ext_sew>.v\t%0,(%3),%4%p1"
1527   [(set_attr "type" "vld<order>x")
1528    (set_attr "mode" "<MODE>")])
1529
1530 (define_insn "@pred_indexed_<order>load<mode>_x8_smaller_eew"
1531   [(set (match_operand:VEEWTRUNC8 0 "register_operand"             "=&vr,  &vr")
1532         (if_then_else:VEEWTRUNC8
1533           (unspec:<VM>
1534             [(match_operand:<VM> 1 "vector_mask_operand"          "vmWc1,vmWc1")
1535              (match_operand 5 "vector_length_operand"             "   rK,   rK")
1536              (match_operand 6 "const_int_operand"                 "    i,    i")
1537              (match_operand 7 "const_int_operand"                 "    i,    i")
1538              (match_operand 8 "const_int_operand"                 "    i,    i")
1539              (reg:SI VL_REGNUM)
1540              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1541           (unspec:VEEWTRUNC8
1542             [(match_operand 3 "pmode_register_operand"            "    r,    r")
1543              (mem:BLK (scratch))
1544              (match_operand:<VINDEX_OCT_EXT> 4 "register_operand" "   vr,   vr")] ORDER)
1545           (match_operand:VEEWTRUNC8 2 "vector_merge_operand"      "   vu,    0")))]
1546   "TARGET_VECTOR"
1547   "vl<order>xei<oct_ext_sew>.v\t%0,(%3),%4%p1"
1548   [(set_attr "type" "vld<order>x")
1549    (set_attr "mode" "<MODE>")])
1550
1551 (define_insn "@pred_indexed_<order>store<VNX1_QHSD:mode><VNX1_QHSDI:mode>"
1552   [(set (mem:BLK (scratch))
1553         (unspec:BLK
1554           [(unspec:<VM>
1555             [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
1556              (match_operand 4 "vector_length_operand"    "   rK")
1557              (match_operand 5 "const_int_operand"        "    i")
1558              (reg:SI VL_REGNUM)
1559              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1560            (match_operand 1 "pmode_register_operand"      "   r")
1561            (match_operand:VNX1_QHSDI 2 "register_operand" "  vr")
1562            (match_operand:VNX1_QHSD 3 "register_operand"  "  vr")] ORDER))]
1563   "TARGET_VECTOR"
1564   "vs<order>xei<VNX1_QHSDI:sew>.v\t%3,(%1),%2%p0"
1565   [(set_attr "type" "vst<order>x")
1566    (set_attr "mode" "<VNX1_QHSD:MODE>")])
1567
1568 (define_insn "@pred_indexed_<order>store<VNX2_QHSD:mode><VNX2_QHSDI:mode>"
1569   [(set (mem:BLK (scratch))
1570         (unspec:BLK
1571           [(unspec:<VM>
1572             [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
1573              (match_operand 4 "vector_length_operand"    "   rK")
1574              (match_operand 5 "const_int_operand"        "    i")
1575              (reg:SI VL_REGNUM)
1576              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1577            (match_operand 1 "pmode_register_operand"      "   r")
1578            (match_operand:VNX2_QHSDI 2 "register_operand" "  vr")
1579            (match_operand:VNX2_QHSD 3 "register_operand"  "  vr")] ORDER))]
1580   "TARGET_VECTOR"
1581   "vs<order>xei<VNX2_QHSDI:sew>.v\t%3,(%1),%2%p0"
1582   [(set_attr "type" "vst<order>x")
1583    (set_attr "mode" "<VNX2_QHSD:MODE>")])
1584
1585 (define_insn "@pred_indexed_<order>store<VNX4_QHSD:mode><VNX4_QHSDI:mode>"
1586   [(set (mem:BLK (scratch))
1587         (unspec:BLK
1588           [(unspec:<VM>
1589             [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
1590              (match_operand 4 "vector_length_operand"    "   rK")
1591              (match_operand 5 "const_int_operand"        "    i")
1592              (reg:SI VL_REGNUM)
1593              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1594            (match_operand 1 "pmode_register_operand"      "   r")
1595            (match_operand:VNX4_QHSDI 2 "register_operand" "  vr")
1596            (match_operand:VNX4_QHSD 3 "register_operand"  "  vr")] ORDER))]
1597   "TARGET_VECTOR"
1598   "vs<order>xei<VNX4_QHSDI:sew>.v\t%3,(%1),%2%p0"
1599   [(set_attr "type" "vst<order>x")
1600    (set_attr "mode" "<VNX4_QHSD:MODE>")])
1601
1602 (define_insn "@pred_indexed_<order>store<VNX8_QHSD:mode><VNX8_QHSDI:mode>"
1603   [(set (mem:BLK (scratch))
1604         (unspec:BLK
1605           [(unspec:<VM>
1606             [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
1607              (match_operand 4 "vector_length_operand"    "   rK")
1608              (match_operand 5 "const_int_operand"        "    i")
1609              (reg:SI VL_REGNUM)
1610              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1611            (match_operand 1 "pmode_register_operand"      "   r")
1612            (match_operand:VNX8_QHSDI 2 "register_operand" "  vr")
1613            (match_operand:VNX8_QHSD 3 "register_operand"  "  vr")] ORDER))]
1614   "TARGET_VECTOR"
1615   "vs<order>xei<VNX8_QHSDI:sew>.v\t%3,(%1),%2%p0"
1616   [(set_attr "type" "vst<order>x")
1617    (set_attr "mode" "<VNX8_QHSD:MODE>")])
1618
1619 (define_insn "@pred_indexed_<order>store<VNX16_QHS:mode><VNX16_QHSI:mode>"
1620   [(set (mem:BLK (scratch))
1621         (unspec:BLK
1622           [(unspec:<VM>
1623             [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
1624              (match_operand 4 "vector_length_operand"    "   rK")
1625              (match_operand 5 "const_int_operand"        "    i")
1626              (reg:SI VL_REGNUM)
1627              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1628            (match_operand 1 "pmode_register_operand"      "   r")
1629            (match_operand:VNX16_QHSI 2 "register_operand" "  vr")
1630            (match_operand:VNX16_QHS 3 "register_operand"  "  vr")] ORDER))]
1631   "TARGET_VECTOR"
1632   "vs<order>xei<VNX16_QHSI:sew>.v\t%3,(%1),%2%p0"
1633   [(set_attr "type" "vst<order>x")
1634    (set_attr "mode" "<VNX16_QHS:MODE>")])
1635
1636 (define_insn "@pred_indexed_<order>store<VNX32_QH:mode><VNX32_QHI:mode>"
1637   [(set (mem:BLK (scratch))
1638         (unspec:BLK
1639           [(unspec:<VM>
1640             [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
1641              (match_operand 4 "vector_length_operand"    "   rK")
1642              (match_operand 5 "const_int_operand"        "    i")
1643              (reg:SI VL_REGNUM)
1644              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1645            (match_operand 1 "pmode_register_operand"      "   r")
1646            (match_operand:VNX32_QHI 2 "register_operand"  "  vr")
1647            (match_operand:VNX32_QH 3 "register_operand"   "  vr")] ORDER))]
1648   "TARGET_VECTOR"
1649   "vs<order>xei<VNX32_QHI:sew>.v\t%3,(%1),%2%p0"
1650   [(set_attr "type" "vst<order>x")
1651    (set_attr "mode" "<VNX32_QH:MODE>")])
1652
1653 (define_insn "@pred_indexed_<order>store<VNX64_Q:mode><VNX64_Q:mode>"
1654   [(set (mem:BLK (scratch))
1655         (unspec:BLK
1656           [(unspec:<VM>
1657             [(match_operand:<VM> 0 "vector_mask_operand" "vmWc1")
1658              (match_operand 4 "vector_length_operand"    "   rK")
1659              (match_operand 5 "const_int_operand"        "    i")
1660              (reg:SI VL_REGNUM)
1661              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1662            (match_operand 1 "pmode_register_operand"      "   r")
1663            (match_operand:VNX64_Q 2 "register_operand"    "  vr")
1664            (match_operand:VNX64_Q 3 "register_operand"    "  vr")] ORDER))]
1665   "TARGET_VECTOR"
1666   "vs<order>xei<VNX64_Q:sew>.v\t%3,(%1),%2%p0"
1667   [(set_attr "type" "vst<order>x")
1668    (set_attr "mode" "<VNX64_Q:MODE>")])
1669
1670 ;; -------------------------------------------------------------------------------
1671 ;; ---- Predicated integer binary operations
1672 ;; -------------------------------------------------------------------------------
1673 ;; Includes:
1674 ;; - 11.1 Vector Single-Width Integer Add and Subtract
1675 ;; - 11.4 Vector Integer Add-with-Carry/Subtract-with-Borrow Instructions
1676 ;; - 11.5 Vector Bitwise Logical Instructions
1677 ;; - 11.6 Vector Single-Width Bit Shift Instructions
1678 ;; - 11.9 Vector Integer Min/Max Instructions
1679 ;; - 11.10 Vector Single-Width Integer Multiply Instructions
1680 ;; - 11.11 Vector Integer Divide Instructions
1681 ;; -------------------------------------------------------------------------------
1682
1683 (define_insn "@pred_<optab><mode>"
1684   [(set (match_operand:VI 0 "register_operand"           "=vd, vd, vr, vr, vd, vd, vr, vr, vd, vd, vr, vr")
1685         (if_then_else:VI
1686           (unspec:<VM>
1687             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1, Wc1, vm, vm,Wc1,Wc1, vm, vm,Wc1,Wc1")
1688              (match_operand 5 "vector_length_operand"    " rK, rK, rK,  rK, rK, rK, rK, rK, rK, rK, rK, rK")
1689              (match_operand 6 "const_int_operand"        "  i,  i,  i,   i,  i,  i,  i,  i,  i,  i,  i,  i")
1690              (match_operand 7 "const_int_operand"        "  i,  i,  i,   i,  i,  i,  i,  i,  i,  i,  i,  i")
1691              (match_operand 8 "const_int_operand"        "  i,  i,  i,   i,  i,  i,  i,  i,  i,  i,  i,  i")
1692              (reg:SI VL_REGNUM)
1693              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1694           (any_int_binop:VI
1695             (match_operand:VI 3 "<binop_rhs1_predicate>" "<binop_rhs1_constraint>")
1696             (match_operand:VI 4 "<binop_rhs2_predicate>" "<binop_rhs2_constraint>"))
1697           (match_operand:VI 2 "vector_merge_operand"     "vu,0,vu,0,vu,0,vu,0,vu,0,vu,0")))]
1698   "TARGET_VECTOR"
1699   "@
1700    v<insn>.vv\t%0,%3,%4%p1
1701    v<insn>.vv\t%0,%3,%4%p1
1702    v<insn>.vv\t%0,%3,%4%p1
1703    v<insn>.vv\t%0,%3,%4%p1
1704    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1
1705    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1
1706    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1
1707    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1
1708    v<binop_reverse_vi_variant_insn>\t%0,<binop_reverse_vi_variant_op>%p1
1709    v<binop_reverse_vi_variant_insn>\t%0,<binop_reverse_vi_variant_op>%p1
1710    v<binop_reverse_vi_variant_insn>\t%0,<binop_reverse_vi_variant_op>%p1
1711    v<binop_reverse_vi_variant_insn>\t%0,<binop_reverse_vi_variant_op>%p1"
1712   [(set_attr "type" "<int_binop_insn_type>")
1713    (set_attr "mode" "<MODE>")])
1714
1715 ;; vx instructions patterns.
1716 ;; Note: Unlike vv patterns, we should split them since they are variant.
1717 ;; For vsll.vx/vsra.vx/vsrl.vx the scalar mode should be Pmode wheras the
1718 ;; scalar mode is inner mode of the RVV mode for other vx patterns.
1719 (define_insn "@pred_<optab><mode>_scalar"
1720   [(set (match_operand:VI 0 "register_operand"           "=vd,vd, vr, vr,vd,vd, vr, vr")
1721         (if_then_else:VI
1722           (unspec:<VM>
1723             [(match_operand:<VM> 1 "vector_mask_operand"  "vm,vm,Wc1,Wc1,vm,vm,Wc1,Wc1")
1724              (match_operand 5 "vector_length_operand"     "rK,rK, rK, rK,rK,rK, rK, rK")
1725              (match_operand 6 "const_int_operand"         " i, i,  i,  i, i, i,  i,  i")
1726              (match_operand 7 "const_int_operand"         " i, i,  i,  i, i, i,  i,  i")
1727              (match_operand 8 "const_int_operand"         " i, i,  i,  i, i, i,  i,  i")
1728              (reg:SI VL_REGNUM)
1729              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1730           (any_shift:VI
1731             (match_operand:VI 3 "register_operand"        "vr,vr, vr, vr,vr,vr, vr, vr")
1732             (match_operand 4 "pmode_reg_or_uimm5_operand" " r, r,  r,  r, K, K,  K,  K"))
1733           (match_operand:VI 2 "vector_merge_operand"      "vu, 0, vu,  0,vu, 0, vu,  0")))]
1734   "TARGET_VECTOR"
1735   "v<insn>.v%o4\t%0,%3,%4%p1"
1736   [(set_attr "type" "vshift")
1737    (set_attr "mode" "<MODE>")])
1738
1739 ;; Handle GET_MODE_INNER (mode) = QImode, HImode, SImode.
1740 (define_insn "@pred_<optab><mode>_scalar"
1741   [(set (match_operand:VI_QHS 0 "register_operand"      "=vd,vd, vr, vr")
1742         (if_then_else:VI_QHS
1743           (unspec:<VM>
1744             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
1745              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
1746              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
1747              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
1748              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
1749              (reg:SI VL_REGNUM)
1750              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1751           (any_commutative_binop:VI_QHS
1752             (vec_duplicate:VI_QHS
1753               (match_operand:<VEL> 4 "reg_or_0_operand"  "rJ,rJ, rJ, rJ"))
1754             (match_operand:VI_QHS 3 "register_operand"   "vr,vr, vr, vr"))
1755           (match_operand:VI_QHS 2 "vector_merge_operand" "vu, 0, vu,  0")))]
1756   "TARGET_VECTOR"
1757   "v<insn>.vx\t%0,%3,%z4%p1"
1758   [(set_attr "type" "<int_binop_insn_type>")
1759    (set_attr "mode" "<MODE>")])
1760
1761 (define_insn "@pred_<optab><mode>_scalar"
1762   [(set (match_operand:VI_QHS 0 "register_operand"      "=vd,vd, vr, vr")
1763         (if_then_else:VI_QHS
1764           (unspec:<VM>
1765             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
1766              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
1767              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
1768              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
1769              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
1770              (reg:SI VL_REGNUM)
1771              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1772           (any_non_commutative_binop:VI_QHS
1773             (match_operand:VI_QHS 3 "register_operand"   "vr,vr, vr, vr")
1774             (vec_duplicate:VI_QHS
1775               (match_operand:<VEL> 4 "reg_or_0_operand"  "rJ,rJ, rJ, rJ")))
1776           (match_operand:VI_QHS 2 "vector_merge_operand" "vu, 0, vu,  0")))]
1777   "TARGET_VECTOR"
1778   "v<insn>.vx\t%0,%3,%z4%p1"
1779   [(set_attr "type" "<int_binop_insn_type>")
1780    (set_attr "mode" "<MODE>")])
1781
1782 (define_insn "@pred_sub<mode>_reverse_scalar"
1783   [(set (match_operand:VI_QHS 0 "register_operand"      "=vd,vd, vr, vr")
1784         (if_then_else:VI_QHS
1785           (unspec:<VM>
1786             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
1787              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
1788              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
1789              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
1790              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
1791              (reg:SI VL_REGNUM)
1792              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1793           (minus:VI_QHS
1794             (vec_duplicate:VI_QHS
1795               (match_operand:<VEL> 4 "reg_or_0_operand"  "rJ,rJ, rJ, rJ"))
1796             (match_operand:VI_QHS 3 "register_operand"   "vr,vr, vr, vr"))
1797           (match_operand:VI_QHS 2 "vector_merge_operand" "vu, 0, vu,  0")))]
1798   "TARGET_VECTOR"
1799   "vrsub.vx\t%0,%3,%z4%p1"
1800   [(set_attr "type" "vialu")
1801    (set_attr "mode" "<MODE>")])
1802
1803 ;; Handle GET_MODE_INNER (mode) = DImode. We need to split them since
1804 ;; we need to deal with SEW = 64 in RV32 system.
1805 (define_expand "@pred_<optab><mode>_scalar"
1806   [(set (match_operand:VI_D 0 "register_operand")
1807         (if_then_else:VI_D
1808           (unspec:<VM>
1809             [(match_operand:<VM> 1 "vector_mask_operand")
1810              (match_operand 5 "vector_length_operand")
1811              (match_operand 6 "const_int_operand")
1812              (match_operand 7 "const_int_operand")
1813              (match_operand 8 "const_int_operand")
1814              (reg:SI VL_REGNUM)
1815              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1816           (any_commutative_binop:VI_D
1817             (vec_duplicate:VI_D
1818               (match_operand:<VEL> 4 "reg_or_int_operand"))
1819             (match_operand:VI_D 3 "register_operand"))
1820           (match_operand:VI_D 2 "vector_merge_operand")))]
1821   "TARGET_VECTOR"
1822 {
1823   if (riscv_vector::sew64_scalar_helper (
1824         operands,
1825         /* scalar op */&operands[4],
1826         /* vl */operands[5],
1827         <MODE>mode,
1828         <VM>mode,
1829         riscv_vector::has_vi_variant_p (<CODE>, operands[4]),
1830         [] (rtx *operands, rtx boardcast_scalar) {
1831           emit_insn (gen_pred_<optab><mode> (operands[0], operands[1],
1832                operands[2], operands[3], boardcast_scalar, operands[5],
1833                operands[6], operands[7], operands[8]));
1834         }))
1835     DONE;
1836 })
1837
1838 (define_insn "*pred_<optab><mode>_scalar"
1839   [(set (match_operand:VI_D 0 "register_operand"         "=vd,vd, vr, vr")
1840         (if_then_else:VI_D
1841           (unspec:<VM>
1842             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
1843              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
1844              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
1845              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
1846              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
1847              (reg:SI VL_REGNUM)
1848              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1849           (any_commutative_binop:VI_D
1850             (vec_duplicate:VI_D
1851               (match_operand:<VEL> 4 "reg_or_0_operand"  "rJ,rJ, rJ, rJ"))
1852             (match_operand:VI_D 3 "register_operand"     "vr,vr, vr, vr"))
1853           (match_operand:VI_D 2 "vector_merge_operand"   "vu, 0, vu,  0")))]
1854   "TARGET_VECTOR"
1855   "v<insn>.vx\t%0,%3,%z4%p1"
1856   [(set_attr "type" "<int_binop_insn_type>")
1857    (set_attr "mode" "<MODE>")])
1858
1859 (define_insn "*pred_<optab><mode>_extended_scalar"
1860   [(set (match_operand:VI_D 0 "register_operand"             "=vd,vd, vr, vr")
1861         (if_then_else:VI_D
1862           (unspec:<VM>
1863             [(match_operand:<VM> 1 "vector_mask_operand"     "vm,vm,Wc1,Wc1")
1864              (match_operand 5 "vector_length_operand"        "rK,rK, rK, rK")
1865              (match_operand 6 "const_int_operand"            " i, i,  i,  i")
1866              (match_operand 7 "const_int_operand"            " i, i,  i,  i")
1867              (match_operand 8 "const_int_operand"            " i, i,  i,  i")
1868              (reg:SI VL_REGNUM)
1869              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1870           (any_commutative_binop:VI_D
1871             (vec_duplicate:VI_D
1872               (sign_extend:<VEL>
1873                 (match_operand:<VSUBEL> 4 "reg_or_0_operand" "rJ,rJ, rJ, rJ")))
1874             (match_operand:VI_D 3 "register_operand"         "vr,vr, vr, vr"))
1875           (match_operand:VI_D 2 "vector_merge_operand"       "vu, 0, vu,  0")))]
1876   "TARGET_VECTOR"
1877   "v<insn>.vx\t%0,%3,%z4%p1"
1878   [(set_attr "type" "<int_binop_insn_type>")
1879    (set_attr "mode" "<MODE>")])
1880
1881 (define_expand "@pred_<optab><mode>_scalar"
1882   [(set (match_operand:VI_D 0 "register_operand")
1883         (if_then_else:VI_D
1884           (unspec:<VM>
1885             [(match_operand:<VM> 1 "vector_mask_operand")
1886              (match_operand 5 "vector_length_operand")
1887              (match_operand 6 "const_int_operand")
1888              (match_operand 7 "const_int_operand")
1889              (match_operand 8 "const_int_operand")
1890              (reg:SI VL_REGNUM)
1891              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1892           (any_non_commutative_binop:VI_D
1893             (match_operand:VI_D 3 "register_operand")
1894             (vec_duplicate:VI_D
1895               (match_operand:<VEL> 4 "reg_or_int_operand")))
1896           (match_operand:VI_D 2 "vector_merge_operand")))]
1897   "TARGET_VECTOR"
1898 {
1899   if (riscv_vector::sew64_scalar_helper (
1900         operands,
1901         /* scalar op */&operands[4],
1902         /* vl */operands[5],
1903         <MODE>mode,
1904         <VM>mode,
1905         riscv_vector::has_vi_variant_p (<CODE>, operands[4]),
1906         [] (rtx *operands, rtx boardcast_scalar) {
1907           emit_insn (gen_pred_<optab><mode> (operands[0], operands[1],
1908                operands[2], operands[3], boardcast_scalar, operands[5],
1909                operands[6], operands[7], operands[8]));
1910         }))
1911     DONE;
1912 })
1913
1914 (define_insn "*pred_<optab><mode>_scalar"
1915   [(set (match_operand:VI_D 0 "register_operand"         "=vd,vd, vr, vr")
1916         (if_then_else:VI_D
1917           (unspec:<VM>
1918             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
1919              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
1920              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
1921              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
1922              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
1923              (reg:SI VL_REGNUM)
1924              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1925           (any_non_commutative_binop:VI_D
1926             (match_operand:VI_D 3 "register_operand"     "vr,vr, vr, vr")
1927             (vec_duplicate:VI_D
1928               (match_operand:<VEL> 4 "reg_or_0_operand"  "rJ,rJ, rJ, rJ")))
1929           (match_operand:VI_D 2 "vector_merge_operand"   "vu, 0, vu,  0")))]
1930   "TARGET_VECTOR"
1931   "v<insn>.vx\t%0,%3,%z4%p1"
1932   [(set_attr "type" "<int_binop_insn_type>")
1933    (set_attr "mode" "<MODE>")])
1934
1935 (define_insn "*pred_<optab><mode>_extended_scalar"
1936   [(set (match_operand:VI_D 0 "register_operand"             "=vd,vd, vr, vr")
1937         (if_then_else:VI_D
1938           (unspec:<VM>
1939             [(match_operand:<VM> 1 "vector_mask_operand"     "vm,vm,Wc1,Wc1")
1940              (match_operand 5 "vector_length_operand"        "rK,rK, rK, rK")
1941              (match_operand 6 "const_int_operand"            " i, i,  i,  i")
1942              (match_operand 7 "const_int_operand"            " i, i,  i,  i")
1943              (match_operand 8 "const_int_operand"            " i, i,  i,  i")
1944              (reg:SI VL_REGNUM)
1945              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1946           (any_non_commutative_binop:VI_D
1947             (match_operand:VI_D 3 "register_operand"         "vr,vr, vr, vr")
1948             (vec_duplicate:VI_D
1949               (sign_extend:<VEL>
1950                 (match_operand:<VSUBEL> 4 "reg_or_0_operand" "rJ,rJ, rJ, rJ"))))
1951           (match_operand:VI_D 2 "vector_merge_operand"       "vu, 0, vu,  0")))]
1952   "TARGET_VECTOR"
1953   "v<insn>.vx\t%0,%3,%z4%p1"
1954   [(set_attr "type" "<int_binop_insn_type>")
1955    (set_attr "mode" "<MODE>")])
1956
1957 (define_expand "@pred_sub<mode>_reverse_scalar"
1958   [(set (match_operand:VI_D 0 "register_operand")
1959         (if_then_else:VI_D
1960           (unspec:<VM>
1961             [(match_operand:<VM> 1 "vector_mask_operand")
1962              (match_operand 5 "vector_length_operand")
1963              (match_operand 6 "const_int_operand")
1964              (match_operand 7 "const_int_operand")
1965              (match_operand 8 "const_int_operand")
1966              (reg:SI VL_REGNUM)
1967              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
1968           (minus:VI_D
1969             (vec_duplicate:VI_D
1970               (match_operand:<VEL> 4 "reg_or_int_operand"))
1971             (match_operand:VI_D 3 "register_operand"))
1972           (match_operand:VI_D 2 "vector_merge_operand")))]
1973   "TARGET_VECTOR"
1974 {
1975   if (riscv_vector::sew64_scalar_helper (
1976         operands,
1977         /* scalar op */&operands[4],
1978         /* vl */operands[5],
1979         <MODE>mode,
1980         <VM>mode,
1981         riscv_vector::neg_simm5_p (operands[4]),
1982         [] (rtx *operands, rtx boardcast_scalar) {
1983           emit_insn (gen_pred_sub<mode> (operands[0], operands[1],
1984                operands[2], boardcast_scalar, operands[3], operands[5],
1985                operands[6], operands[7], operands[8]));
1986         }))
1987     DONE;
1988 })
1989
1990 (define_insn "*pred_sub<mode>_reverse_scalar"
1991   [(set (match_operand:VI_D 0 "register_operand"         "=vd,vd, vr, vr")
1992         (if_then_else:VI_D
1993           (unspec:<VM>
1994             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
1995              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
1996              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
1997              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
1998              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
1999              (reg:SI VL_REGNUM)
2000              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2001           (minus:VI_D
2002             (vec_duplicate:VI_D
2003               (match_operand:<VEL> 4 "reg_or_0_operand"  "rJ,rJ, rJ, rJ"))
2004             (match_operand:VI_D 3 "register_operand"     "vr,vr, vr, vr"))
2005           (match_operand:VI_D 2 "vector_merge_operand"   "vu, 0, vu,  0")))]
2006   "TARGET_VECTOR"
2007   "vrsub.vx\t%0,%3,%z4%p1"
2008   [(set_attr "type" "vialu")
2009    (set_attr "mode" "<MODE>")])
2010
2011 (define_insn "*pred_sub<mode>_extended_reverse_scalar"
2012   [(set (match_operand:VI_D 0 "register_operand"             "=vd,vd, vr, vr")
2013         (if_then_else:VI_D
2014           (unspec:<VM>
2015             [(match_operand:<VM> 1 "vector_mask_operand"     "vm,vm,Wc1,Wc1")
2016              (match_operand 5 "vector_length_operand"        "rK,rK, rK, rK")
2017              (match_operand 6 "const_int_operand"            " i, i,  i,  i")
2018              (match_operand 7 "const_int_operand"            " i, i,  i,  i")
2019              (match_operand 8 "const_int_operand"            " i, i,  i,  i")
2020              (reg:SI VL_REGNUM)
2021              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2022           (minus:VI_D
2023             (vec_duplicate:VI_D
2024               (sign_extend:<VEL>
2025                 (match_operand:<VSUBEL> 4 "reg_or_0_operand" "rJ,rJ, rJ, rJ")))
2026             (match_operand:VI_D 3 "register_operand"         "vr,vr, vr, vr"))
2027           (match_operand:VI_D 2 "vector_merge_operand"       "vu, 0, vu,  0")))]
2028   "TARGET_VECTOR"
2029   "vrsub.vx\t%0,%3,%z4%p1"
2030   [(set_attr "type" "vialu")
2031    (set_attr "mode" "<MODE>")])
2032
2033 ;; Multiply High instructions.
2034 (define_insn "@pred_mulh<v_su><mode>"
2035   [(set (match_operand:VFULLI 0 "register_operand"       "=vd,vd, vr, vr")
2036         (if_then_else:VFULLI
2037           (unspec:<VM>
2038             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
2039              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
2040              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
2041              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
2042              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
2043              (reg:SI VL_REGNUM)
2044              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2045           (unspec:VFULLI
2046             [(match_operand:VFULLI 3 "register_operand"  "vr,vr, vr, vr")
2047              (match_operand:VFULLI 4 "register_operand"  "vr,vr, vr, vr")] VMULH)
2048           (match_operand:VFULLI 2 "vector_merge_operand" "vu, 0, vu,  0")))]
2049   "TARGET_VECTOR"
2050   "vmulh<v_su>.vv\t%0,%3,%4%p1"
2051   [(set_attr "type" "vimul")
2052    (set_attr "mode" "<MODE>")])
2053
2054 (define_insn "@pred_mulh<v_su><mode>_scalar"
2055   [(set (match_operand:VI_QHS 0 "register_operand"       "=vd,vd, vr, vr")
2056         (if_then_else:VI_QHS
2057           (unspec:<VM>
2058             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
2059              (match_operand 5 "vector_length_operand"    "rK,rK, rK, rK")
2060              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
2061              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
2062              (match_operand 8 "const_int_operand"        " i, i,  i,  i")
2063              (reg:SI VL_REGNUM)
2064              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2065           (unspec:VI_QHS
2066             [(vec_duplicate:VI_QHS
2067                (match_operand:<VEL> 4 "reg_or_0_operand"  "rJ,rJ, rJ, rJ"))
2068              (match_operand:VI_QHS 3 "register_operand"   "vr,vr, vr, vr")] VMULH)
2069           (match_operand:VI_QHS 2 "vector_merge_operand"  "vu, 0, vu,  0")))]
2070   "TARGET_VECTOR"
2071   "vmulh<v_su>.vx\t%0,%3,%z4%p1"
2072   [(set_attr "type" "vimul")
2073    (set_attr "mode" "<MODE>")])
2074
2075 (define_expand "@pred_mulh<v_su><mode>_scalar"
2076   [(set (match_operand:VFULLI_D 0 "register_operand")
2077         (if_then_else:VFULLI_D
2078           (unspec:<VM>
2079             [(match_operand:<VM> 1 "vector_mask_operand")
2080              (match_operand 5 "vector_length_operand")
2081              (match_operand 6 "const_int_operand")
2082              (match_operand 7 "const_int_operand")
2083              (match_operand 8 "const_int_operand")
2084              (reg:SI VL_REGNUM)
2085              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2086           (unspec:VFULLI_D
2087             [(vec_duplicate:VFULLI_D
2088                (match_operand:<VEL> 4 "reg_or_int_operand"))
2089              (match_operand:VFULLI_D 3 "register_operand")] VMULH)
2090           (match_operand:VFULLI_D 2 "vector_merge_operand")))]
2091   "TARGET_VECTOR"
2092 {
2093   if (riscv_vector::sew64_scalar_helper (
2094         operands,
2095         /* scalar op */&operands[4],
2096         /* vl */operands[5],
2097         <MODE>mode,
2098         <VM>mode,
2099         false,
2100         [] (rtx *operands, rtx boardcast_scalar) {
2101           emit_insn (gen_pred_mulh<v_su><mode> (operands[0], operands[1],
2102                operands[2], operands[3], boardcast_scalar, operands[5],
2103                operands[6], operands[7], operands[8]));
2104         }))
2105     DONE;
2106 })
2107
2108 (define_insn "*pred_mulh<v_su><mode>_scalar"
2109   [(set (match_operand:VFULLI_D 0 "register_operand"       "=vd,vd, vr, vr")
2110         (if_then_else:VFULLI_D
2111           (unspec:<VM>
2112             [(match_operand:<VM> 1 "vector_mask_operand"   "vm,vm,Wc1,Wc1")
2113              (match_operand 5 "vector_length_operand"      "rK,rK, rK, rK")
2114              (match_operand 6 "const_int_operand"          " i, i,  i,  i")
2115              (match_operand 7 "const_int_operand"          " i, i,  i,  i")
2116              (match_operand 8 "const_int_operand"          " i, i,  i,  i")
2117              (reg:SI VL_REGNUM)
2118              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2119           (unspec:VFULLI_D
2120             [(vec_duplicate:VFULLI_D
2121                (match_operand:<VEL> 4 "reg_or_0_operand"   "rJ,rJ, rJ, rJ"))
2122              (match_operand:VFULLI_D 3 "register_operand"  "vr,vr, vr, vr")] VMULH)
2123           (match_operand:VFULLI_D 2 "vector_merge_operand" "vu, 0, vu,  0")))]
2124   "TARGET_VECTOR"
2125   "vmulh<v_su>.vx\t%0,%3,%z4%p1"
2126   [(set_attr "type" "vimul")
2127    (set_attr "mode" "<MODE>")])
2128
2129 (define_insn "*pred_mulh<v_su><mode>_extended_scalar"
2130   [(set (match_operand:VFULLI_D 0 "register_operand"          "=vd,vd, vr, vr")
2131         (if_then_else:VFULLI_D
2132           (unspec:<VM>
2133             [(match_operand:<VM> 1 "vector_mask_operand"      "vm,vm,Wc1,Wc1")
2134              (match_operand 5 "vector_length_operand"         "rK,rK, rK, rK")
2135              (match_operand 6 "const_int_operand"             " i, i,  i,  i")
2136              (match_operand 7 "const_int_operand"             " i, i,  i,  i")
2137              (match_operand 8 "const_int_operand"             " i, i,  i,  i")
2138              (reg:SI VL_REGNUM)
2139              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2140           (unspec:VFULLI_D
2141             [(vec_duplicate:VFULLI_D
2142                (sign_extend:<VEL>
2143                  (match_operand:<VSUBEL> 4 "reg_or_0_operand" "rJ,rJ, rJ, rJ")))
2144              (match_operand:VFULLI_D 3 "register_operand"     "vr,vr, vr, vr")] VMULH)
2145           (match_operand:VFULLI_D 2 "vector_merge_operand"    "vu, 0, vu,  0")))]
2146   "TARGET_VECTOR"
2147   "vmulh<v_su>.vx\t%0,%3,%z4%p1"
2148   [(set_attr "type" "vimul")
2149    (set_attr "mode" "<MODE>")])
2150
2151 ;; Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
2152 (define_insn "@pred_adc<mode>"
2153   [(set (match_operand:VI 0 "register_operand"           "=vd,vd,vd,vd")
2154         (if_then_else:VI
2155           (unspec:<VM>
2156             [(match_operand 5 "vector_length_operand"     "rK,rK,rK,rK")
2157              (match_operand 6 "const_int_operand"         " i, i, i, i")
2158              (match_operand 7 "const_int_operand"         " i, i, i, i")
2159              (reg:SI VL_REGNUM)
2160              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2161           (unspec:VI
2162              [(plus:VI
2163                (match_operand:VI 2 "register_operand"     "vr,vr,vr,vr")
2164                (match_operand:VI 3 "vector_arith_operand" "vr,vr,vi,vi"))
2165              (match_operand:<VM> 4 "register_operand"     "vm,vm,vm,vm")] UNSPEC_VADC)
2166           (match_operand:VI 1 "vector_merge_operand"      "vu, 0,vu, 0")))]
2167   "TARGET_VECTOR"
2168   "vadc.v%o3m\t%0,%2,%v3,%4"
2169   [(set_attr "type" "vicalu")
2170    (set_attr "mode" "<MODE>")
2171    (set_attr "merge_op_idx" "1")
2172    (set_attr "vl_op_idx" "5")
2173    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2174    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2175
2176 (define_insn "@pred_sbc<mode>"
2177   [(set (match_operand:VI 0 "register_operand"           "=vd,vd")
2178         (if_then_else:VI
2179           (unspec:<VM>
2180             [(match_operand 5 "vector_length_operand"     "rK,rK")
2181              (match_operand 6 "const_int_operand"         " i, i")
2182              (match_operand 7 "const_int_operand"         " i, i")
2183              (reg:SI VL_REGNUM)
2184              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2185           (unspec:VI
2186              [(minus:VI
2187                (match_operand:VI 2 "register_operand"     "vr,vr")
2188                (match_operand:VI 3 "register_operand"     "vr,vr"))
2189               (match_operand:<VM> 4 "register_operand"    "vm,vm")] UNSPEC_VSBC)
2190           (match_operand:VI 1 "vector_merge_operand"      "vu, 0")))]
2191   "TARGET_VECTOR"
2192   "vsbc.vvm\t%0,%2,%3,%4"
2193   [(set_attr "type" "vicalu")
2194    (set_attr "mode" "<MODE>")
2195    (set_attr "merge_op_idx" "1")
2196    (set_attr "vl_op_idx" "5")
2197    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2198    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2199
2200 (define_insn "@pred_adc<mode>_scalar"
2201   [(set (match_operand:VI_QHS 0 "register_operand"        "=vd,vd")
2202         (if_then_else:VI_QHS
2203           (unspec:<VM>
2204             [(match_operand 5 "vector_length_operand"      "rK,rK")
2205              (match_operand 6 "const_int_operand"          " i, i")
2206              (match_operand 7 "const_int_operand"          " i, i")
2207              (reg:SI VL_REGNUM)
2208              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2209           (unspec:VI_QHS
2210              [(plus:VI_QHS
2211                (vec_duplicate:VI_QHS
2212                  (match_operand:<VEL> 3 "register_operand" " r, r"))
2213                (match_operand:VI_QHS 2 "register_operand"  "vr,vr"))
2214              (match_operand:<VM> 4 "register_operand"      "vm,vm")] UNSPEC_VADC)
2215           (match_operand:VI_QHS 1 "vector_merge_operand"   "vu, 0")))]
2216   "TARGET_VECTOR"
2217   "vadc.vxm\t%0,%2,%3,%4"
2218   [(set_attr "type" "vicalu")
2219    (set_attr "mode" "<MODE>")
2220    (set_attr "merge_op_idx" "1")
2221    (set_attr "vl_op_idx" "5")
2222    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2223    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2224
2225 (define_insn "@pred_sbc<mode>_scalar"
2226   [(set (match_operand:VI_QHS 0 "register_operand"         "=vd,vd")
2227         (if_then_else:VI_QHS
2228           (unspec:<VM>
2229             [(match_operand 5 "vector_length_operand"       "rK,rK")
2230              (match_operand 6 "const_int_operand"           " i, i")
2231              (match_operand 7 "const_int_operand"           " i, i")
2232              (reg:SI VL_REGNUM)
2233              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2234           (unspec:VI_QHS
2235              [(minus:VI_QHS
2236                 (match_operand:VI_QHS 2 "register_operand"  "vr,vr")
2237                 (vec_duplicate:VI_QHS
2238                   (match_operand:<VEL> 3 "reg_or_0_operand" "rJ,rJ")))
2239               (match_operand:<VM> 4 "register_operand"      "vm,vm")] UNSPEC_VSBC)
2240           (match_operand:VI_QHS 1 "vector_merge_operand"    "vu, 0")))]
2241   "TARGET_VECTOR"
2242   "vsbc.vxm\t%0,%2,%z3,%4"
2243   [(set_attr "type" "vicalu")
2244    (set_attr "mode" "<MODE>")
2245    (set_attr "merge_op_idx" "1")
2246    (set_attr "vl_op_idx" "5")
2247    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2248    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2249
2250 (define_expand "@pred_adc<mode>_scalar"
2251   [(set (match_operand:VI_D 0 "register_operand")
2252         (if_then_else:VI_D
2253           (unspec:<VM>
2254             [(match_operand 5 "vector_length_operand")
2255              (match_operand 6 "const_int_operand")
2256              (match_operand 7 "const_int_operand")
2257              (reg:SI VL_REGNUM)
2258              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2259           (unspec:VI_D
2260              [(plus:VI_D
2261                 (vec_duplicate:VI_D
2262                   (match_operand:<VEL> 3 "reg_or_int_operand"))
2263                 (match_operand:VI_D 2 "register_operand"))
2264               (match_operand:<VM> 4 "register_operand")] UNSPEC_VADC)
2265           (match_operand:VI_D 1 "vector_merge_operand")))]
2266   "TARGET_VECTOR"
2267 {
2268   if (riscv_vector::sew64_scalar_helper (
2269         operands,
2270         /* scalar op */&operands[3],
2271         /* vl */operands[5],
2272         <MODE>mode,
2273         <VM>mode,
2274         riscv_vector::simm5_p (operands[3]),
2275         [] (rtx *operands, rtx boardcast_scalar) {
2276           emit_insn (gen_pred_adc<mode> (operands[0], operands[1],
2277                operands[2], boardcast_scalar, operands[4], operands[5],
2278                operands[6], operands[7]));
2279         }))
2280     DONE;
2281 })
2282
2283 (define_insn "*pred_adc<mode>_scalar"
2284   [(set (match_operand:VI_D 0 "register_operand"           "=vd,vd")
2285         (if_then_else:VI_D
2286           (unspec:<VM>
2287             [(match_operand 5 "vector_length_operand"       "rK,rK")
2288              (match_operand 6 "const_int_operand"           " i, i")
2289              (match_operand 7 "const_int_operand"           " i, i")
2290              (reg:SI VL_REGNUM)
2291              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2292           (unspec:VI_D
2293              [(plus:VI_D
2294                 (vec_duplicate:VI_D
2295                   (match_operand:<VEL> 3 "reg_or_0_operand" "rJ,rJ"))
2296                 (match_operand:VI_D 2 "register_operand"    "vr,vr"))
2297               (match_operand:<VM> 4 "register_operand"      "vm,vm")] UNSPEC_VADC)
2298           (match_operand:VI_D 1 "vector_merge_operand"      "vu, 0")))]
2299   "TARGET_VECTOR"
2300   "vadc.vxm\t%0,%2,%z3,%4"
2301   [(set_attr "type" "vicalu")
2302    (set_attr "mode" "<MODE>")
2303    (set_attr "merge_op_idx" "1")
2304    (set_attr "vl_op_idx" "5")
2305    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2306    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2307
2308 (define_insn "*pred_adc<mode>_extended_scalar"
2309   [(set (match_operand:VI_D 0 "register_operand"                "=vd,vd")
2310         (if_then_else:VI_D
2311           (unspec:<VM>
2312             [(match_operand 5 "vector_length_operand"            "rK,rK")
2313              (match_operand 6 "const_int_operand"                " i, i")
2314              (match_operand 7 "const_int_operand"                " i, i")
2315              (reg:SI VL_REGNUM)
2316              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2317           (unspec:VI_D
2318              [(plus:VI_D
2319                 (vec_duplicate:VI_D
2320                   (sign_extend:<VEL>
2321                     (match_operand:<VSUBEL> 3 "reg_or_0_operand" "rJ,rJ")))
2322                 (match_operand:VI_D 2 "register_operand"         "vr,vr"))
2323               (match_operand:<VM> 4 "register_operand"           "vm,vm")] UNSPEC_VADC)
2324           (match_operand:VI_D 1 "vector_merge_operand"           "vu, 0")))]
2325   "TARGET_VECTOR"
2326   "vadc.vxm\t%0,%2,%z3,%4"
2327   [(set_attr "type" "vicalu")
2328    (set_attr "mode" "<MODE>")
2329    (set_attr "merge_op_idx" "1")
2330    (set_attr "vl_op_idx" "5")
2331    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2332    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2333
2334 (define_expand "@pred_sbc<mode>_scalar"
2335   [(set (match_operand:VI_D 0 "register_operand")
2336         (if_then_else:VI_D
2337           (unspec:<VM>
2338             [(match_operand 5 "vector_length_operand")
2339              (match_operand 6 "const_int_operand")
2340              (match_operand 7 "const_int_operand")
2341              (reg:SI VL_REGNUM)
2342              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2343           (unspec:VI_D
2344              [(minus:VI_D
2345                 (match_operand:VI_D 2 "register_operand")
2346                 (vec_duplicate:VI_D
2347                   (match_operand:<VEL> 3 "reg_or_int_operand")))
2348               (match_operand:<VM> 4 "register_operand")] UNSPEC_VSBC)
2349           (match_operand:VI_D 1 "vector_merge_operand")))]
2350   "TARGET_VECTOR"
2351 {
2352   if (riscv_vector::sew64_scalar_helper (
2353         operands,
2354         /* scalar op */&operands[3],
2355         /* vl */operands[5],
2356         <MODE>mode,
2357         <VM>mode,
2358         false,
2359         [] (rtx *operands, rtx boardcast_scalar) {
2360           emit_insn (gen_pred_sbc<mode> (operands[0], operands[1],
2361                operands[2], boardcast_scalar, operands[4], operands[5],
2362                operands[6], operands[7]));
2363         }))
2364     DONE;
2365 })
2366
2367 (define_insn "*pred_sbc<mode>_scalar"
2368   [(set (match_operand:VI_D 0 "register_operand"           "=vd,vd")
2369         (if_then_else:VI_D
2370           (unspec:<VM>
2371             [(match_operand 5 "vector_length_operand"       "rK,rK")
2372              (match_operand 6 "const_int_operand"           " i, i")
2373              (match_operand 7 "const_int_operand"           " i, i")
2374              (reg:SI VL_REGNUM)
2375              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2376           (unspec:VI_D
2377              [(minus:VI_D
2378                 (match_operand:VI_D 2 "register_operand"    "vr,vr")
2379                 (vec_duplicate:VI_D
2380                   (match_operand:<VEL> 3 "reg_or_0_operand" "rJ,rJ")))
2381               (match_operand:<VM> 4 "register_operand"      "vm,vm")] UNSPEC_VSBC)
2382           (match_operand:VI_D 1 "vector_merge_operand"      "vu, 0")))]
2383   "TARGET_VECTOR"
2384   "vsbc.vxm\t%0,%2,%z3,%4"
2385   [(set_attr "type" "vicalu")
2386    (set_attr "mode" "<MODE>")
2387    (set_attr "merge_op_idx" "1")
2388    (set_attr "vl_op_idx" "5")
2389    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2390    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2391
2392 (define_insn "*pred_sbc<mode>_extended_scalar"
2393   [(set (match_operand:VI_D 0 "register_operand"                "=vd,vd")
2394         (if_then_else:VI_D
2395           (unspec:<VM>
2396             [(match_operand 5 "vector_length_operand"           "rK,rK")
2397              (match_operand 6 "const_int_operand"               " i, i")
2398              (match_operand 7 "const_int_operand"               " i, i")
2399              (reg:SI VL_REGNUM)
2400              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2401           (unspec:VI_D
2402              [(minus:VI_D
2403                 (match_operand:VI_D 2 "register_operand"         "vr,vr")
2404                 (vec_duplicate:VI_D
2405                   (sign_extend:<VEL>
2406                     (match_operand:<VSUBEL> 3 "reg_or_0_operand" "rJ,rJ"))))
2407               (match_operand:<VM> 4 "register_operand"           "vm,vm")] UNSPEC_VSBC)
2408           (match_operand:VI_D 1 "vector_merge_operand"           "vu, 0")))]
2409   "TARGET_VECTOR"
2410   "vsbc.vxm\t%0,%2,%z3,%4"
2411   [(set_attr "type" "vicalu")
2412    (set_attr "mode" "<MODE>")
2413    (set_attr "merge_op_idx" "1")
2414    (set_attr "vl_op_idx" "5")
2415    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
2416    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2417
2418 (define_insn "@pred_madc<mode>"
2419   [(set (match_operand:<VM> 0 "register_operand"        "=&vr, &vr")
2420         (unspec:<VM>
2421            [(plus:VI
2422              (match_operand:VI 1 "register_operand"     "  vr,  vr")
2423              (match_operand:VI 2 "vector_arith_operand" "  vr,  vi"))
2424             (match_operand:<VM> 3 "register_operand"    "  vm,  vm")
2425             (unspec:<VM>
2426               [(match_operand 4 "vector_length_operand" "  rK,  rK")
2427                (match_operand 5 "const_int_operand"     "   i,   i")
2428                (reg:SI VL_REGNUM)
2429                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMADC))]
2430   "TARGET_VECTOR"
2431   "vmadc.v%o2m\t%0,%1,%v2,%3"
2432   [(set_attr "type" "vicalu")
2433    (set_attr "mode" "<MODE>")
2434    (set_attr "vl_op_idx" "4")
2435    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2436
2437 (define_insn "@pred_msbc<mode>"
2438   [(set (match_operand:<VM> 0 "register_operand"        "=&vr")
2439         (unspec:<VM>
2440            [(minus:VI
2441              (match_operand:VI 1 "register_operand"     "  vr")
2442              (match_operand:VI 2 "register_operand"     "  vr"))
2443             (match_operand:<VM> 3 "register_operand"    "  vm")
2444             (unspec:<VM>
2445               [(match_operand 4 "vector_length_operand" "  rK")
2446                (match_operand 5 "const_int_operand"     "   i")
2447                (reg:SI VL_REGNUM)
2448                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMSBC))]
2449   "TARGET_VECTOR"
2450   "vmsbc.vvm\t%0,%1,%2,%3"
2451   [(set_attr "type" "vicalu")
2452    (set_attr "mode" "<MODE>")
2453    (set_attr "vl_op_idx" "4")
2454    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2455
2456 (define_insn "@pred_madc<mode>_scalar"
2457   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2458         (unspec:<VM>
2459            [(plus:VI_QHS
2460              (vec_duplicate:VI_QHS
2461                (match_operand:<VEL> 2 "register_operand" "   r"))
2462              (match_operand:VI_QHS 1 "register_operand"  "  vr"))
2463             (match_operand:<VM> 3 "register_operand"     "  vm")
2464             (unspec:<VM>
2465               [(match_operand 4 "vector_length_operand"  "  rK")
2466                (match_operand 5 "const_int_operand"      "   i")
2467                (reg:SI VL_REGNUM)
2468                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMADC))]
2469   "TARGET_VECTOR"
2470   "vmadc.vxm\t%0,%1,%2,%3"
2471   [(set_attr "type" "vicalu")
2472    (set_attr "mode" "<MODE>")
2473    (set_attr "vl_op_idx" "4")
2474    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2475
2476 (define_insn "@pred_msbc<mode>_scalar"
2477   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2478         (unspec:<VM>
2479            [(minus:VI_QHS
2480              (vec_duplicate:VI_QHS
2481                (match_operand:<VEL> 2 "reg_or_0_operand" "  rJ"))
2482              (match_operand:VI_QHS 1 "register_operand"  "  vr"))
2483             (match_operand:<VM> 3 "register_operand"     "  vm")
2484             (unspec:<VM>
2485               [(match_operand 4 "vector_length_operand"  "  rK")
2486                (match_operand 5 "const_int_operand"      "   i")
2487                (reg:SI VL_REGNUM)
2488                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMSBC))]
2489   "TARGET_VECTOR"
2490   "vmsbc.vxm\t%0,%1,%z2,%3"
2491   [(set_attr "type" "vicalu")
2492    (set_attr "mode" "<MODE>")
2493    (set_attr "vl_op_idx" "4")
2494    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2495
2496 (define_expand "@pred_madc<mode>_scalar"
2497   [(set (match_operand:<VM> 0 "register_operand")
2498         (unspec:<VM>
2499            [(plus:VI_D
2500              (vec_duplicate:VI_D
2501                (match_operand:<VEL> 2 "reg_or_int_operand"))
2502              (match_operand:VI_D 1 "register_operand"))
2503             (match_operand:<VM> 3 "register_operand")
2504             (unspec:<VM>
2505               [(match_operand 4 "vector_length_operand")
2506                (match_operand 5 "const_int_operand")
2507                (reg:SI VL_REGNUM)
2508                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMADC))]
2509   "TARGET_VECTOR"
2510 {
2511   if (riscv_vector::sew64_scalar_helper (
2512         operands,
2513         /* scalar op */&operands[2],
2514         /* vl */operands[4],
2515         <MODE>mode,
2516         <VM>mode,
2517         riscv_vector::simm5_p (operands[2]),
2518         [] (rtx *operands, rtx boardcast_scalar) {
2519           emit_insn (gen_pred_madc<mode> (operands[0], operands[1],
2520                boardcast_scalar, operands[3], operands[4], operands[5]));
2521         }))
2522     DONE;
2523 })
2524
2525 (define_insn "*pred_madc<mode>_scalar"
2526   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2527         (unspec:<VM>
2528            [(plus:VI_D
2529              (vec_duplicate:VI_D
2530                (match_operand:<VEL> 2 "reg_or_0_operand" "  rJ"))
2531              (match_operand:VI_D 1 "register_operand"    "  vr"))
2532             (match_operand:<VM> 3 "register_operand"     "  vm")
2533             (unspec:<VM>
2534               [(match_operand 4 "vector_length_operand"  "  rK")
2535                (match_operand 5 "const_int_operand"      "   i")
2536                (reg:SI VL_REGNUM)
2537                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMADC))]
2538   "TARGET_VECTOR"
2539   "vmadc.vxm\t%0,%1,%z2,%3"
2540   [(set_attr "type" "vicalu")
2541    (set_attr "mode" "<MODE>")
2542    (set_attr "vl_op_idx" "4")
2543    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2544
2545 (define_insn "*pred_madc<mode>_extended_scalar"
2546   [(set (match_operand:<VM> 0 "register_operand"             "=&vr")
2547         (unspec:<VM>
2548            [(plus:VI_D
2549              (vec_duplicate:VI_D
2550                (sign_extend:<VEL>
2551                  (match_operand:<VSUBEL> 2 "reg_or_0_operand" "  rJ")))
2552              (match_operand:VI_D 1 "register_operand"         "  vr"))
2553             (match_operand:<VM> 3 "register_operand"          "  vm")
2554             (unspec:<VM>
2555               [(match_operand 4 "vector_length_operand"       "  rK")
2556                (match_operand 5 "const_int_operand"           "   i")
2557                (reg:SI VL_REGNUM)
2558                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMADC))]
2559   "TARGET_VECTOR"
2560   "vmadc.vxm\t%0,%1,%z2,%3"
2561   [(set_attr "type" "vicalu")
2562    (set_attr "mode" "<MODE>")
2563    (set_attr "vl_op_idx" "4")
2564    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2565
2566 (define_expand "@pred_msbc<mode>_scalar"
2567   [(set (match_operand:<VM> 0 "register_operand")
2568         (unspec:<VM>
2569            [(minus:VI_D
2570              (vec_duplicate:VI_D
2571                (match_operand:<VEL> 2 "reg_or_int_operand"))
2572              (match_operand:VI_D 1 "register_operand"))
2573             (match_operand:<VM> 3 "register_operand")
2574             (unspec:<VM>
2575               [(match_operand 4 "vector_length_operand")
2576                (match_operand 5 "const_int_operand")
2577                (reg:SI VL_REGNUM)
2578                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMSBC))]
2579   "TARGET_VECTOR"
2580 {
2581   if (riscv_vector::sew64_scalar_helper (
2582         operands,
2583         /* scalar op */&operands[2],
2584         /* vl */operands[4],
2585         <MODE>mode,
2586         <VM>mode,
2587         false,
2588         [] (rtx *operands, rtx boardcast_scalar) {
2589           emit_insn (gen_pred_msbc<mode> (operands[0], operands[1],
2590                boardcast_scalar, operands[3], operands[4], operands[5]));
2591         }))
2592     DONE;
2593 })
2594
2595 (define_insn "*pred_msbc<mode>_scalar"
2596   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2597         (unspec:<VM>
2598            [(minus:VI_D
2599              (vec_duplicate:VI_D
2600                (match_operand:<VEL> 2 "reg_or_0_operand" "  rJ"))
2601              (match_operand:VI_D 1 "register_operand"    "  vr"))
2602             (match_operand:<VM> 3 "register_operand"     "  vm")
2603             (unspec:<VM>
2604               [(match_operand 4 "vector_length_operand"  "  rK")
2605                (match_operand 5 "const_int_operand"      "   i")
2606                (reg:SI VL_REGNUM)
2607                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMSBC))]
2608   "TARGET_VECTOR"
2609   "vmsbc.vxm\t%0,%1,%z2,%3"
2610   [(set_attr "type" "vicalu")
2611    (set_attr "mode" "<MODE>")
2612    (set_attr "vl_op_idx" "4")
2613    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2614
2615 (define_insn "*pred_msbc<mode>_extended_scalar"
2616   [(set (match_operand:<VM> 0 "register_operand"             "=&vr")
2617         (unspec:<VM>
2618            [(minus:VI_D
2619              (vec_duplicate:VI_D
2620                (sign_extend:<VEL>
2621                  (match_operand:<VSUBEL> 2 "reg_or_0_operand" "  rJ")))
2622              (match_operand:VI_D 1 "register_operand"         "  vr"))
2623             (match_operand:<VM> 3 "register_operand"          "  vm")
2624             (unspec:<VM>
2625               [(match_operand 4 "vector_length_operand"       "  rK")
2626                (match_operand 5 "const_int_operand"           "   i")
2627                (reg:SI VL_REGNUM)
2628                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_VMSBC))]
2629   "TARGET_VECTOR"
2630   "vmsbc.vxm\t%0,%1,%z2,%3"
2631   [(set_attr "type" "vicalu")
2632    (set_attr "mode" "<MODE>")
2633    (set_attr "vl_op_idx" "4")
2634    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
2635
2636 (define_insn "@pred_madc<mode>_overflow"
2637   [(set (match_operand:<VM> 0 "register_operand"        "=&vr, &vr")
2638         (unspec:<VM>
2639            [(plus:VI
2640              (match_operand:VI 1 "register_operand"     "  vr,  vr")
2641              (match_operand:VI 2 "vector_arith_operand" "  vr,  vi"))
2642             (unspec:<VM>
2643               [(match_operand 3 "vector_length_operand" "  rK,  rK")
2644                (match_operand 4 "const_int_operand"     "   i,   i")
2645                (reg:SI VL_REGNUM)
2646                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2647   "TARGET_VECTOR"
2648   "vmadc.v%o2\t%0,%1,%v2"
2649   [(set_attr "type" "vicalu")
2650    (set_attr "mode" "<MODE>")
2651    (set_attr "vl_op_idx" "3")
2652    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2653
2654 (define_insn "@pred_msbc<mode>_overflow"
2655   [(set (match_operand:<VM> 0 "register_operand"        "=&vr")
2656         (unspec:<VM>
2657            [(minus:VI
2658              (match_operand:VI 1 "register_operand"     "  vr")
2659              (match_operand:VI 2 "register_operand"     "  vr"))
2660             (unspec:<VM>
2661               [(match_operand 3 "vector_length_operand" "  rK")
2662                (match_operand 4 "const_int_operand"     "   i")
2663                (reg:SI VL_REGNUM)
2664                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2665   "TARGET_VECTOR"
2666   "vmsbc.vv\t%0,%1,%2"
2667   [(set_attr "type" "vicalu")
2668    (set_attr "mode" "<MODE>")
2669    (set_attr "vl_op_idx" "3")
2670    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2671
2672 (define_insn "@pred_madc<mode>_overflow_scalar"
2673   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2674         (unspec:<VM>
2675            [(plus:VI_QHS
2676              (vec_duplicate:VI_QHS
2677                (match_operand:<VEL> 2 "reg_or_0_operand" "  rJ"))
2678              (match_operand:VI_QHS 1 "register_operand"  "  vr"))
2679             (unspec:<VM>
2680               [(match_operand 3 "vector_length_operand"  "  rK")
2681                (match_operand 4 "const_int_operand"      "   i")
2682                (reg:SI VL_REGNUM)
2683                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2684   "TARGET_VECTOR"
2685   "vmadc.vx\t%0,%1,%z2"
2686   [(set_attr "type" "vicalu")
2687    (set_attr "mode" "<MODE>")
2688    (set_attr "vl_op_idx" "3")
2689    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2690
2691 (define_insn "@pred_msbc<mode>_overflow_scalar"
2692   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2693         (unspec:<VM>
2694            [(minus:VI_QHS
2695              (vec_duplicate:VI_QHS
2696                (match_operand:<VEL> 2 "reg_or_0_operand" "  rJ"))
2697              (match_operand:VI_QHS 1 "register_operand"  "  vr"))
2698             (unspec:<VM>
2699               [(match_operand 3 "vector_length_operand"  "  rK")
2700                (match_operand 4 "const_int_operand"      "   i")
2701                (reg:SI VL_REGNUM)
2702                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2703   "TARGET_VECTOR"
2704   "vmsbc.vx\t%0,%1,%z2"
2705   [(set_attr "type" "vicalu")
2706    (set_attr "mode" "<MODE>")
2707    (set_attr "vl_op_idx" "3")
2708    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2709
2710 (define_expand "@pred_madc<mode>_overflow_scalar"
2711   [(set (match_operand:<VM> 0 "register_operand")
2712         (unspec:<VM>
2713            [(plus:VI_D
2714              (vec_duplicate:VI_D
2715                (match_operand:<VEL> 2 "reg_or_int_operand"))
2716              (match_operand:VI_D 1 "register_operand"))
2717             (unspec:<VM>
2718               [(match_operand 3 "vector_length_operand")
2719                (match_operand 4 "const_int_operand")
2720                (reg:SI VL_REGNUM)
2721                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2722   "TARGET_VECTOR"
2723 {
2724   if (riscv_vector::sew64_scalar_helper (
2725         operands,
2726         /* scalar op */&operands[2],
2727         /* vl */operands[3],
2728         <MODE>mode,
2729         <VM>mode,
2730         riscv_vector::simm5_p (operands[2]),
2731         [] (rtx *operands, rtx boardcast_scalar) {
2732           emit_insn (gen_pred_madc<mode>_overflow (operands[0], operands[1],
2733                boardcast_scalar, operands[3], operands[4]));
2734         }))
2735     DONE;
2736 })
2737
2738 (define_insn "*pred_madc<mode>_overflow_scalar"
2739   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2740         (unspec:<VM>
2741            [(plus:VI_D
2742              (vec_duplicate:VI_D
2743                (match_operand:<VEL> 2 "reg_or_0_operand" "  rJ"))
2744              (match_operand:VI_D 1 "register_operand"    "  vr"))
2745             (unspec:<VM>
2746               [(match_operand 3 "vector_length_operand"  "  rK")
2747                (match_operand 4 "const_int_operand"      "   i")
2748                (reg:SI VL_REGNUM)
2749                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2750   "TARGET_VECTOR"
2751   "vmadc.vx\t%0,%1,%z2"
2752   [(set_attr "type" "vicalu")
2753    (set_attr "mode" "<MODE>")
2754    (set_attr "vl_op_idx" "3")
2755    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2756
2757 (define_insn "*pred_madc<mode>_overflow_extended_scalar"
2758   [(set (match_operand:<VM> 0 "register_operand"             "=&vr")
2759         (unspec:<VM>
2760            [(plus:VI_D
2761              (vec_duplicate:VI_D
2762                (sign_extend:<VEL>
2763                  (match_operand:<VSUBEL> 2 "reg_or_0_operand" "  rJ")))
2764              (match_operand:VI_D 1 "register_operand"         "  vr"))
2765             (unspec:<VM>
2766               [(match_operand 3 "vector_length_operand"       "  rK")
2767                (match_operand 4 "const_int_operand"           "   i")
2768                (reg:SI VL_REGNUM)
2769                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2770   "TARGET_VECTOR"
2771   "vmadc.vx\t%0,%1,%z2"
2772   [(set_attr "type" "vicalu")
2773    (set_attr "mode" "<MODE>")
2774    (set_attr "vl_op_idx" "3")
2775    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2776
2777 (define_expand "@pred_msbc<mode>_overflow_scalar"
2778   [(set (match_operand:<VM> 0 "register_operand")
2779         (unspec:<VM>
2780            [(minus:VI_D
2781              (vec_duplicate:VI_D
2782                (match_operand:<VEL> 2 "reg_or_int_operand"))
2783              (match_operand:VI_D 1 "register_operand"))
2784             (unspec:<VM>
2785               [(match_operand 3 "vector_length_operand")
2786                (match_operand 4 "const_int_operand")
2787                (reg:SI VL_REGNUM)
2788                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2789   "TARGET_VECTOR"
2790 {
2791   if (riscv_vector::sew64_scalar_helper (
2792         operands,
2793         /* scalar op */&operands[2],
2794         /* vl */operands[3],
2795         <MODE>mode,
2796         <VM>mode,
2797         false,
2798         [] (rtx *operands, rtx boardcast_scalar) {
2799           emit_insn (gen_pred_msbc<mode>_overflow (operands[0], operands[1],
2800                boardcast_scalar, operands[3], operands[4]));
2801         }))
2802     DONE;
2803 })
2804
2805 (define_insn "*pred_msbc<mode>_overflow_scalar"
2806   [(set (match_operand:<VM> 0 "register_operand"         "=&vr")
2807         (unspec:<VM>
2808            [(minus:VI_D
2809              (vec_duplicate:VI_D
2810                (match_operand:<VEL> 2 "reg_or_0_operand" "  rJ"))
2811              (match_operand:VI_D 1 "register_operand"    "  vr"))
2812             (unspec:<VM>
2813               [(match_operand 3 "vector_length_operand"  "  rK")
2814                (match_operand 4 "const_int_operand"      "   i")
2815                (reg:SI VL_REGNUM)
2816                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2817   "TARGET_VECTOR"
2818   "vmsbc.vx\t%0,%1,%z2"
2819   [(set_attr "type" "vicalu")
2820    (set_attr "mode" "<MODE>")
2821    (set_attr "vl_op_idx" "3")
2822    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2823
2824 (define_insn "*pred_msbc<mode>_overflow_extended_scalar"
2825   [(set (match_operand:<VM> 0 "register_operand"             "=&vr")
2826         (unspec:<VM>
2827            [(minus:VI_D
2828              (vec_duplicate:VI_D
2829                (sign_extend:<VEL>
2830                  (match_operand:<VSUBEL> 2 "reg_or_0_operand" "  rJ")))
2831              (match_operand:VI_D 1 "register_operand"         "  vr"))
2832             (unspec:<VM>
2833               [(match_operand 3 "vector_length_operand"       "  rK")
2834                (match_operand 4 "const_int_operand"           "   i")
2835                (reg:SI VL_REGNUM)
2836                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)] UNSPEC_OVERFLOW))]
2837   "TARGET_VECTOR"
2838   "vmsbc.vx\t%0,%1,%z2"
2839   [(set_attr "type" "vicalu")
2840    (set_attr "mode" "<MODE>")
2841    (set_attr "vl_op_idx" "3")
2842    (set (attr "avl_type") (symbol_ref "INTVAL (operands[4])"))])
2843
2844 ;; -------------------------------------------------------------------------------
2845 ;; ---- Predicated integer unary operations
2846 ;; -------------------------------------------------------------------------------
2847 ;; Includes:
2848 ;; - vneg.v/vnot.v
2849 ;; -------------------------------------------------------------------------------
2850
2851 (define_insn "@pred_<optab><mode>"
2852   [(set (match_operand:VI 0 "register_operand"          "=vd,vd, vr, vr")
2853         (if_then_else:VI
2854           (unspec:<VM>
2855             [(match_operand:<VM> 1 "vector_mask_operand" "vm,vm,Wc1,Wc1")
2856              (match_operand 4 "vector_length_operand"    "rK,rK, rK, rK")
2857              (match_operand 5 "const_int_operand"        " i, i,  i,  i")
2858              (match_operand 6 "const_int_operand"        " i, i,  i,  i")
2859              (match_operand 7 "const_int_operand"        " i, i,  i,  i")
2860              (reg:SI VL_REGNUM)
2861              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2862           (any_int_unop:VI
2863             (match_operand:VI 3 "register_operand"       "vr,vr, vr, vr"))
2864           (match_operand:VI 2 "vector_merge_operand"     "vu, 0, vu,  0")))]
2865   "TARGET_VECTOR"
2866   "v<insn>.v\t%0,%3%p1"
2867   [(set_attr "type" "vialu")
2868    (set_attr "mode" "<MODE>")
2869    (set_attr "vl_op_idx" "4")
2870    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[5])"))
2871    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[6])"))
2872    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
2873
2874 ;; -------------------------------------------------------------------------------
2875 ;; ---- Predicated integer widening binary operations
2876 ;; -------------------------------------------------------------------------------
2877 ;; Includes:
2878 ;; - 11.2 Vector Widening Integer Add/Subtract
2879 ;; - 11.3 Vector Integer Extension
2880 ;; - 11.12 Vector Widening Integer Multiply Instructions
2881 ;; -------------------------------------------------------------------------------
2882
2883 ;; Vector Double-Widening Sign-extend and Zero-extend.
2884 (define_insn "@pred_<optab><mode>_vf2"
2885   [(set (match_operand:VWEXTI 0 "register_operand"            "=&vr,&vr")
2886         (if_then_else:VWEXTI
2887           (unspec:<VM>
2888             [(match_operand:<VM> 1 "vector_mask_operand"         "vmWc1,vmWc1")
2889              (match_operand 4 "vector_length_operand"            "   rK,   rK")
2890              (match_operand 5 "const_int_operand"                "    i,    i")
2891              (match_operand 6 "const_int_operand"                "    i,    i")
2892              (match_operand 7 "const_int_operand"                "    i,    i")
2893              (reg:SI VL_REGNUM)
2894              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2895           (any_extend:VWEXTI
2896             (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
2897           (match_operand:VWEXTI 2 "vector_merge_operand"         "   vu,    0")))]
2898   "TARGET_VECTOR"
2899   "v<sz>ext.vf2\t%0,%3%p1"
2900   [(set_attr "type" "vext")
2901    (set_attr "mode" "<MODE>")])
2902
2903 ;; Vector Quad-Widening Sign-extend and Zero-extend.
2904 (define_insn "@pred_<optab><mode>_vf4"
2905   [(set (match_operand:VQEXTI 0 "register_operand"          "=&vr,&vr")
2906         (if_then_else:VQEXTI
2907           (unspec:<VM>
2908             [(match_operand:<VM> 1 "vector_mask_operand"       "vmWc1,vmWc1")
2909              (match_operand 4 "vector_length_operand"          "   rK,   rK")
2910              (match_operand 5 "const_int_operand"              "    i,    i")
2911              (match_operand 6 "const_int_operand"              "    i,    i")
2912              (match_operand 7 "const_int_operand"              "    i,    i")
2913              (reg:SI VL_REGNUM)
2914              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2915           (any_extend:VQEXTI
2916             (match_operand:<V_QUAD_TRUNC> 3 "register_operand" "   vr,   vr"))
2917           (match_operand:VQEXTI 2 "vector_merge_operand"       "   vu,    0")))]
2918   "TARGET_VECTOR"
2919   "v<sz>ext.vf4\t%0,%3%p1"
2920   [(set_attr "type" "vext")
2921    (set_attr "mode" "<MODE>")])
2922
2923 ;; Vector Oct-Widening Sign-extend and Zero-extend.
2924 (define_insn "@pred_<optab><mode>_vf8"
2925   [(set (match_operand:VOEXTI 0 "register_operand"         "=&vr,&vr")
2926         (if_then_else:VOEXTI
2927           (unspec:<VM>
2928             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
2929              (match_operand 4 "vector_length_operand"         "   rK,   rK")
2930              (match_operand 5 "const_int_operand"             "    i,    i")
2931              (match_operand 6 "const_int_operand"             "    i,    i")
2932              (match_operand 7 "const_int_operand"             "    i,    i")
2933              (reg:SI VL_REGNUM)
2934              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2935           (any_extend:VOEXTI
2936             (match_operand:<V_OCT_TRUNC> 3 "register_operand" "   vr,   vr"))
2937           (match_operand:VOEXTI 2 "vector_merge_operand"      "   vu,    0")))]
2938   "TARGET_VECTOR"
2939   "v<sz>ext.vf8\t%0,%3%p1"
2940   [(set_attr "type" "vext")
2941    (set_attr "mode" "<MODE>")])
2942
2943 ;; Vector Widening Add/Subtract/Multiply.
2944 (define_insn "@pred_dual_widen_<any_widen_binop:optab><any_extend:su><mode>"
2945   [(set (match_operand:VWEXTI 0 "register_operand"                  "=&vr,&vr")
2946         (if_then_else:VWEXTI
2947           (unspec:<VM>
2948             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
2949              (match_operand 5 "vector_length_operand"              "   rK,   rK")
2950              (match_operand 6 "const_int_operand"                  "    i,    i")
2951              (match_operand 7 "const_int_operand"                  "    i,    i")
2952              (match_operand 8 "const_int_operand"                  "    i,    i")
2953              (reg:SI VL_REGNUM)
2954              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2955           (any_widen_binop:VWEXTI
2956             (any_extend:VWEXTI
2957               (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
2958             (any_extend:VWEXTI
2959               (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr,   vr")))
2960           (match_operand:VWEXTI 2 "vector_merge_operand"           "   vu,    0")))]
2961   "TARGET_VECTOR"
2962   "vw<any_widen_binop:insn><any_extend:u>.vv\t%0,%3,%4%p1"
2963   [(set_attr "type" "vi<widen_binop_insn_type>")
2964    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
2965
2966 (define_insn "@pred_dual_widen_<any_widen_binop:optab><any_extend:su><mode>_scalar"
2967   [(set (match_operand:VWEXTI 0 "register_operand"                  "=&vr,&vr")
2968         (if_then_else:VWEXTI
2969           (unspec:<VM>
2970             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
2971              (match_operand 5 "vector_length_operand"              "   rK,   rK")
2972              (match_operand 6 "const_int_operand"                  "    i,    i")
2973              (match_operand 7 "const_int_operand"                  "    i,    i")
2974              (match_operand 8 "const_int_operand"                  "    i,    i")
2975              (reg:SI VL_REGNUM)
2976              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
2977           (any_widen_binop:VWEXTI
2978             (any_extend:VWEXTI
2979               (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
2980             (any_extend:VWEXTI
2981               (vec_duplicate:<V_DOUBLE_TRUNC>
2982                 (match_operand:<VSUBEL> 4 "reg_or_0_operand"       "   rJ,   rJ"))))
2983           (match_operand:VWEXTI 2 "vector_merge_operand"           "   vu,    0")))]
2984   "TARGET_VECTOR"
2985   "vw<any_widen_binop:insn><any_extend:u>.vx\t%0,%3,%z4%p1"
2986   [(set_attr "type" "vi<widen_binop_insn_type>")
2987    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
2988
2989 (define_insn "@pred_single_widen_<plus_minus:optab><any_extend:su><mode>"
2990   [(set (match_operand:VWEXTI 0 "register_operand"                  "=&vr,&vr")
2991         (if_then_else:VWEXTI
2992           (unspec:<VM>
2993             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
2994              (match_operand 5 "vector_length_operand"              "   rK,   rK")
2995              (match_operand 6 "const_int_operand"                  "    i,    i")
2996              (match_operand 7 "const_int_operand"                  "    i,    i")
2997              (match_operand 8 "const_int_operand"                  "    i,    i")
2998              (reg:SI VL_REGNUM)
2999              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3000           (plus_minus:VWEXTI
3001             (match_operand:VWEXTI 3 "register_operand"             "   vr,   vr")
3002             (any_extend:VWEXTI
3003               (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr,   vr")))
3004           (match_operand:VWEXTI 2 "vector_merge_operand"           "   vu,    0")))]
3005   "TARGET_VECTOR"
3006   "vw<plus_minus:insn><any_extend:u>.wv\t%0,%3,%4%p1"
3007   [(set_attr "type" "vi<widen_binop_insn_type>")
3008    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3009
3010 (define_insn "@pred_single_widen_<plus_minus:optab><any_extend:su><mode>_scalar"
3011   [(set (match_operand:VWEXTI 0 "register_operand"                  "=&vr,&vr")
3012         (if_then_else:VWEXTI
3013           (unspec:<VM>
3014             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
3015              (match_operand 5 "vector_length_operand"              "   rK,   rK")
3016              (match_operand 6 "const_int_operand"                  "    i,    i")
3017              (match_operand 7 "const_int_operand"                  "    i,    i")
3018              (match_operand 8 "const_int_operand"                  "    i,    i")
3019              (reg:SI VL_REGNUM)
3020              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3021           (plus_minus:VWEXTI
3022             (match_operand:VWEXTI 3 "register_operand"             "   vr,   vr")
3023             (any_extend:VWEXTI
3024               (vec_duplicate:<V_DOUBLE_TRUNC>
3025                 (match_operand:<VSUBEL> 4 "reg_or_0_operand"       "   rJ,   rJ"))))
3026           (match_operand:VWEXTI 2 "vector_merge_operand"           "   vu,    0")))]
3027   "TARGET_VECTOR"
3028   "vw<plus_minus:insn><any_extend:u>.wx\t%0,%3,%z4%p1"
3029   [(set_attr "type" "vi<widen_binop_insn_type>")
3030    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3031
3032 (define_insn "@pred_widen_mulsu<mode>"
3033   [(set (match_operand:VWEXTI 0 "register_operand"                  "=&vr,&vr")
3034         (if_then_else:VWEXTI
3035           (unspec:<VM>
3036             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
3037              (match_operand 5 "vector_length_operand"              "   rK,   rK")
3038              (match_operand 6 "const_int_operand"                  "    i,    i")
3039              (match_operand 7 "const_int_operand"                  "    i,    i")
3040              (match_operand 8 "const_int_operand"                  "    i,    i")
3041              (reg:SI VL_REGNUM)
3042              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3043           (mult:VWEXTI
3044             (sign_extend:VWEXTI
3045               (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
3046             (zero_extend:VWEXTI
3047               (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr,   vr")))
3048           (match_operand:VWEXTI 2 "vector_merge_operand"           "   vu,    0")))]
3049   "TARGET_VECTOR"
3050   "vwmulsu.vv\t%0,%3,%4%p1"
3051   [(set_attr "type" "viwmul")
3052    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3053
3054 (define_insn "@pred_widen_mulsu<mode>_scalar"
3055   [(set (match_operand:VWEXTI 0 "register_operand"                  "=&vr,&vr")
3056         (if_then_else:VWEXTI
3057           (unspec:<VM>
3058             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
3059              (match_operand 5 "vector_length_operand"              "   rK,   rK")
3060              (match_operand 6 "const_int_operand"                  "    i,    i")
3061              (match_operand 7 "const_int_operand"                  "    i,    i")
3062              (match_operand 8 "const_int_operand"                  "    i,    i")
3063              (reg:SI VL_REGNUM)
3064              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3065           (mult:VWEXTI
3066             (sign_extend:VWEXTI
3067               (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
3068             (zero_extend:VWEXTI
3069               (vec_duplicate:<V_DOUBLE_TRUNC>
3070                 (match_operand:<VSUBEL> 4 "reg_or_0_operand"       "   rJ,   rJ"))))
3071           (match_operand:VWEXTI 2 "vector_merge_operand"           "   vu,    0")))]
3072   "TARGET_VECTOR"
3073   "vwmulsu.vx\t%0,%3,%z4%p1"
3074   [(set_attr "type" "viwmul")
3075    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3076
3077 ;; vwcvt<u>.x.x.v
3078 (define_insn "@pred_<optab><mode>"
3079   [(set (match_operand:VWEXTI 0 "register_operand"                  "=&vr,&vr")
3080         (if_then_else:VWEXTI
3081           (unspec:<VM>
3082             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
3083              (match_operand 4 "vector_length_operand"              "   rK,   rK")
3084              (match_operand 5 "const_int_operand"                  "    i,    i")
3085              (match_operand 6 "const_int_operand"                  "    i,    i")
3086              (match_operand 7 "const_int_operand"                  "    i,    i")
3087              (reg:SI VL_REGNUM)
3088              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3089           (plus:VWEXTI
3090             (any_extend:VWEXTI
3091               (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
3092             (vec_duplicate:VWEXTI
3093               (reg:<VEL> X0_REGNUM)))
3094           (match_operand:VWEXTI 2 "vector_merge_operand"           "   vu,    0")))]
3095   "TARGET_VECTOR"
3096   "vwcvt<u>.x.x.v\t%0,%3%p1"
3097   [(set_attr "type" "viwalu")
3098    (set_attr "mode" "<V_DOUBLE_TRUNC>")
3099    (set_attr "vl_op_idx" "4")
3100    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[5])"))
3101    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[6])"))
3102    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
3103
3104 ;; -------------------------------------------------------------------------------
3105 ;; ---- Predicated integer Narrowing operations
3106 ;; -------------------------------------------------------------------------------
3107 ;; Includes:
3108 ;; - 11.7 Vector Narrowing Integer Right Shift Instructions
3109 ;; -------------------------------------------------------------------------------
3110
3111 ;; The destination EEW is smaller than the source EEW and the overlap is in the
3112 ;; lowest-numbered part of the source register group
3113 ;; e.g, when LMUL = 1, vnsrl.wi v0,v0,3 is legal but a destination of v1 is not.
3114 (define_insn "@pred_narrow_<optab><mode>"
3115   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"           "=vd,vd, vr, vr,vd, vr,  &vr,  &vr, vd, vr,  &vr,  &vr")
3116         (if_then_else:<V_DOUBLE_TRUNC>
3117           (unspec:<VM>
3118             [(match_operand:<VM> 1 "vector_mask_operand"               " vm,vm,Wc1,Wc1,vm,Wc1,vmWc1,vmWc1, vm,Wc1,vmWc1,vmWc1")
3119              (match_operand 5 "vector_length_operand"                  " rK,rK, rK, rK,rK, rK,   rK,   rK, rK, rK,   rK,   rK")
3120              (match_operand 6 "const_int_operand"                      "  i, i,  i,  i, i,  i,    i,    i,  i,  i,    i,    i")
3121              (match_operand 7 "const_int_operand"                      "  i, i,  i,  i, i,  i,    i,    i,  i,  i,    i,    i")
3122              (match_operand 8 "const_int_operand"                      "  i, i,  i,  i, i,  i,    i,    i,  i,  i,    i,    i")
3123              (reg:SI VL_REGNUM)
3124              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3125           (truncate:<V_DOUBLE_TRUNC>
3126             (any_shiftrt:VWEXTI
3127              (match_operand:VWEXTI 3 "register_operand"                " vr,vr, vr, vr, 0,  0,   vr,   vr,  0,  0,   vr,   vr")
3128              (match_operand:<V_DOUBLE_TRUNC> 4 "vector_shift_operand"  "  0, 0,  0,  0,vr, vr,   vr,   vr, vk, vk,   vk,   vk")))
3129           (match_operand:<V_DOUBLE_TRUNC> 2 "vector_merge_operand"     "  0,vu,  0, vu,vu, vu,   vu,    0, vu, vu,   vu,    0")))]
3130   "TARGET_VECTOR"
3131   "vn<insn>.w%o4\t%0,%3,%v4%p1"
3132   [(set_attr "type" "vnshift")
3133    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3134
3135 (define_insn "@pred_narrow_<optab><mode>_scalar"
3136   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"           "=vd, vd, vr, vr,  &vr,  &vr")
3137         (if_then_else:<V_DOUBLE_TRUNC>
3138           (unspec:<VM>
3139             [(match_operand:<VM> 1 "vector_mask_operand"               " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
3140              (match_operand 5 "vector_length_operand"                  " rK, rK, rK, rK,   rK,   rK")
3141              (match_operand 6 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3142              (match_operand 7 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3143              (match_operand 8 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3144              (reg:SI VL_REGNUM)
3145              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3146           (truncate:<V_DOUBLE_TRUNC>
3147             (any_shiftrt:VWEXTI
3148              (match_operand:VWEXTI 3 "register_operand"                "  0,  0,  0,  0,   vr,   vr")
3149              (match_operand 4 "pmode_reg_or_uimm5_operand"             " rK, rK, rK, rK,   rK,   rK")))
3150           (match_operand:<V_DOUBLE_TRUNC> 2 "vector_merge_operand"     " vu,  0, vu,  0,   vu,    0")))]
3151   "TARGET_VECTOR"
3152   "vn<insn>.w%o4\t%0,%3,%4%p1"
3153   [(set_attr "type" "vnshift")
3154    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3155
3156 ;; vncvt.x.x.w
3157 (define_insn "@pred_trunc<mode>"
3158   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"           "=vd, vd, vr, vr,  &vr,  &vr")
3159         (if_then_else:<V_DOUBLE_TRUNC>
3160           (unspec:<VM>
3161             [(match_operand:<VM> 1 "vector_mask_operand"               " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
3162              (match_operand 4 "vector_length_operand"                  " rK, rK, rK, rK,   rK,   rK")
3163              (match_operand 5 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3164              (match_operand 6 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3165              (match_operand 7 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3166              (reg:SI VL_REGNUM)
3167              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3168           (truncate:<V_DOUBLE_TRUNC>
3169             (match_operand:VWEXTI 3 "register_operand"                 "  0,  0,  0,  0,   vr,   vr"))
3170           (match_operand:<V_DOUBLE_TRUNC> 2 "vector_merge_operand"     " vu,  0, vu,  0,   vu,    0")))]
3171   "TARGET_VECTOR"
3172   "vncvt.x.x.w\t%0,%3%p1"
3173   [(set_attr "type" "vnshift")
3174    (set_attr "mode" "<V_DOUBLE_TRUNC>")
3175    (set_attr "vl_op_idx" "4")
3176    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[5])"))
3177    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[6])"))
3178    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
3179
3180 ;; -------------------------------------------------------------------------------
3181 ;; ---- Predicated fixed-point operations
3182 ;; -------------------------------------------------------------------------------
3183 ;; Includes:
3184 ;; - 12.1 Vector Single-Width Saturating Add and Subtract
3185 ;; - 12.2 Vector Single-Width Aaveraging Add and Subtract
3186 ;; - 12.3 Vector Single-Width Fractional Multiply with Rounding and Saturation
3187 ;; - 12.4 Vector Single-Width Scaling Shift Instructions
3188 ;; - 12.5 Vector Narrowing Fixed-Point Clip Instructions
3189 ;; -------------------------------------------------------------------------------
3190
3191 ;; Saturating Add and Subtract
3192 (define_insn "@pred_<optab><mode>"
3193   [(set (match_operand:VI 0 "register_operand"           "=vd, vd, vr, vr, vd, vd, vr, vr")
3194         (if_then_else:VI
3195           (unspec:<VM>
3196             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1, vm, vm,Wc1,Wc1")
3197              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK, rK, rK, rK, rK")
3198              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i,  i,  i,  i,  i")
3199              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i,  i,  i,  i,  i")
3200              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i,  i,  i,  i,  i")
3201              (reg:SI VL_REGNUM)
3202              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3203           (any_sat_int_binop:VI
3204             (match_operand:VI 3 "<binop_rhs1_predicate>" " vr, vr, vr, vr, vr, vr, vr, vr")
3205             (match_operand:VI 4 "<binop_rhs2_predicate>" "<binop_rhs2_constraint>"))
3206           (match_operand:VI 2 "vector_merge_operand"     " vu,  0, vu,  0, vu,  0, vu,  0")))]
3207   "TARGET_VECTOR"
3208   "@
3209    v<insn>.vv\t%0,%3,%4%p1
3210    v<insn>.vv\t%0,%3,%4%p1
3211    v<insn>.vv\t%0,%3,%4%p1
3212    v<insn>.vv\t%0,%3,%4%p1
3213    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1
3214    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1
3215    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1
3216    v<binop_vi_variant_insn>\t%0,<binop_vi_variant_op>%p1"
3217   [(set_attr "type" "<int_binop_insn_type>")
3218    (set_attr "mode" "<MODE>")])
3219
3220 ;; Handle GET_MODE_INNER (mode) = QImode, HImode, SImode.
3221 (define_insn "@pred_<optab><mode>_scalar"
3222   [(set (match_operand:VI_QHS 0 "register_operand"       "=vd, vd, vr, vr")
3223         (if_then_else:VI_QHS
3224           (unspec:<VM>
3225             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
3226              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
3227              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
3228              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
3229              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
3230              (reg:SI VL_REGNUM)
3231              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3232           (sat_int_plus_binop:VI_QHS
3233             (vec_duplicate:VI_QHS
3234               (match_operand:<VEL> 4 "register_operand"  "  r,  r,  r,  r"))
3235             (match_operand:VI_QHS 3 "register_operand"   " vr, vr, vr, vr"))
3236           (match_operand:VI_QHS 2 "vector_merge_operand" " vu,  0, vu,  0")))]
3237   "TARGET_VECTOR"
3238   "v<insn>.vx\t%0,%3,%4%p1"
3239   [(set_attr "type" "<int_binop_insn_type>")
3240    (set_attr "mode" "<MODE>")])
3241
3242 (define_insn "@pred_<optab><mode>_scalar"
3243   [(set (match_operand:VI_QHS 0 "register_operand"       "=vd, vd, vr, vr")
3244         (if_then_else:VI_QHS
3245           (unspec:<VM>
3246             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
3247              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
3248              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
3249              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
3250              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
3251              (reg:SI VL_REGNUM)
3252              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3253           (sat_int_minus_binop:VI_QHS
3254             (match_operand:VI_QHS 3 "register_operand"   " vr, vr, vr, vr")
3255             (vec_duplicate:VI_QHS
3256               (match_operand:<VEL> 4 "register_operand"  "  r,  r,  r,  r")))
3257           (match_operand:VI_QHS 2 "vector_merge_operand" " vu,  0, vu,  0")))]
3258   "TARGET_VECTOR"
3259   "v<insn>.vx\t%0,%3,%4%p1"
3260   [(set_attr "type" "<int_binop_insn_type>")
3261    (set_attr "mode" "<MODE>")])
3262
3263 (define_expand "@pred_<optab><mode>_scalar"
3264   [(set (match_operand:VI_D 0 "register_operand")
3265         (if_then_else:VI_D
3266           (unspec:<VM>
3267             [(match_operand:<VM> 1 "vector_mask_operand")
3268              (match_operand 5 "vector_length_operand")
3269              (match_operand 6 "const_int_operand")
3270              (match_operand 7 "const_int_operand")
3271              (match_operand 8 "const_int_operand")
3272              (reg:SI VL_REGNUM)
3273              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3274           (sat_int_plus_binop:VI_D
3275             (vec_duplicate:VI_D
3276               (match_operand:<VEL> 4 "reg_or_int_operand"))
3277             (match_operand:VI_D 3 "register_operand"))
3278           (match_operand:VI_D 2 "vector_merge_operand")))]
3279   "TARGET_VECTOR"
3280 {
3281   if (riscv_vector::sew64_scalar_helper (
3282         operands,
3283         /* scalar op */&operands[4],
3284         /* vl */operands[5],
3285         <MODE>mode,
3286         <VM>mode,
3287         riscv_vector::has_vi_variant_p (<CODE>, operands[4]),
3288         [] (rtx *operands, rtx boardcast_scalar) {
3289           emit_insn (gen_pred_<optab><mode> (operands[0], operands[1],
3290                operands[2], operands[3], boardcast_scalar, operands[5],
3291                operands[6], operands[7], operands[8]));
3292         }))
3293     DONE;
3294 })
3295
3296 (define_insn "*pred_<optab><mode>_scalar"
3297   [(set (match_operand:VI_D 0 "register_operand"         "=vd, vd, vr, vr")
3298         (if_then_else:VI_D
3299           (unspec:<VM>
3300             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
3301              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
3302              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
3303              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
3304              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
3305              (reg:SI VL_REGNUM)
3306              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3307           (sat_int_plus_binop:VI_D
3308             (vec_duplicate:VI_D
3309               (match_operand:<VEL> 4 "register_operand"  "  r,  r,  r,  r"))
3310             (match_operand:VI_D 3 "register_operand"     " vr, vr, vr, vr"))
3311           (match_operand:VI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")))]
3312   "TARGET_VECTOR"
3313   "v<insn>.vx\t%0,%3,%4%p1"
3314   [(set_attr "type" "<int_binop_insn_type>")
3315    (set_attr "mode" "<MODE>")])
3316
3317 (define_insn "*pred_<optab><mode>_extended_scalar"
3318   [(set (match_operand:VI_D 0 "register_operand"             "=vd, vd, vr, vr")
3319         (if_then_else:VI_D
3320           (unspec:<VM>
3321             [(match_operand:<VM> 1 "vector_mask_operand"     " vm, vm,Wc1,Wc1")
3322              (match_operand 5 "vector_length_operand"        " rK, rK, rK, rK")
3323              (match_operand 6 "const_int_operand"            "  i,  i,  i,  i")
3324              (match_operand 7 "const_int_operand"            "  i,  i,  i,  i")
3325              (match_operand 8 "const_int_operand"            "  i,  i,  i,  i")
3326              (reg:SI VL_REGNUM)
3327              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3328           (sat_int_plus_binop:VI_D
3329             (vec_duplicate:VI_D
3330               (sign_extend:<VEL>
3331                 (match_operand:<VSUBEL> 4 "register_operand" "  r,  r,  r,  r")))
3332             (match_operand:VI_D 3 "register_operand"         " vr, vr, vr, vr"))
3333           (match_operand:VI_D 2 "vector_merge_operand"       " vu,  0, vu,  0")))]
3334   "TARGET_VECTOR"
3335   "v<insn>.vx\t%0,%3,%4%p1"
3336   [(set_attr "type" "<int_binop_insn_type>")
3337    (set_attr "mode" "<MODE>")])
3338
3339 (define_expand "@pred_<optab><mode>_scalar"
3340   [(set (match_operand:VI_D 0 "register_operand")
3341         (if_then_else:VI_D
3342           (unspec:<VM>
3343             [(match_operand:<VM> 1 "vector_mask_operand")
3344              (match_operand 5 "vector_length_operand")
3345              (match_operand 6 "const_int_operand")
3346              (match_operand 7 "const_int_operand")
3347              (match_operand 8 "const_int_operand")
3348              (reg:SI VL_REGNUM)
3349              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3350           (sat_int_minus_binop:VI_D
3351             (match_operand:VI_D 3 "register_operand")
3352             (vec_duplicate:VI_D
3353               (match_operand:<VEL> 4 "reg_or_int_operand")))
3354           (match_operand:VI_D 2 "vector_merge_operand")))]
3355   "TARGET_VECTOR"
3356 {
3357   if (riscv_vector::sew64_scalar_helper (
3358         operands,
3359         /* scalar op */&operands[4],
3360         /* vl */operands[5],
3361         <MODE>mode,
3362         <VM>mode,
3363         riscv_vector::has_vi_variant_p (<CODE>, operands[4]),
3364         [] (rtx *operands, rtx boardcast_scalar) {
3365           emit_insn (gen_pred_<optab><mode> (operands[0], operands[1],
3366                operands[2], operands[3], boardcast_scalar, operands[5],
3367                operands[6], operands[7], operands[8]));
3368         }))
3369     DONE;
3370 })
3371
3372 (define_insn "*pred_<optab><mode>_scalar"
3373   [(set (match_operand:VI_D 0 "register_operand"         "=vd, vd, vr, vr")
3374         (if_then_else:VI_D
3375           (unspec:<VM>
3376             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
3377              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
3378              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
3379              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
3380              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
3381              (reg:SI VL_REGNUM)
3382              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3383           (sat_int_minus_binop:VI_D
3384             (match_operand:VI_D 3 "register_operand"     " vr, vr, vr, vr")
3385             (vec_duplicate:VI_D
3386               (match_operand:<VEL> 4 "register_operand"  "  r,  r,  r,  r")))
3387           (match_operand:VI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")))]
3388   "TARGET_VECTOR"
3389   "v<insn>.vx\t%0,%3,%4%p1"
3390   [(set_attr "type" "<int_binop_insn_type>")
3391    (set_attr "mode" "<MODE>")])
3392
3393 (define_insn "*pred_<optab><mode>_extended_scalar"
3394   [(set (match_operand:VI_D 0 "register_operand"             "=vd, vd, vr, vr")
3395         (if_then_else:VI_D
3396           (unspec:<VM>
3397             [(match_operand:<VM> 1 "vector_mask_operand"     " vm, vm,Wc1,Wc1")
3398              (match_operand 5 "vector_length_operand"        " rK, rK, rK, rK")
3399              (match_operand 6 "const_int_operand"            "  i,  i,  i,  i")
3400              (match_operand 7 "const_int_operand"            "  i,  i,  i,  i")
3401              (match_operand 8 "const_int_operand"            "  i,  i,  i,  i")
3402              (reg:SI VL_REGNUM)
3403              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3404           (sat_int_minus_binop:VI_D
3405             (match_operand:VI_D 3 "register_operand"         " vr, vr, vr, vr")
3406             (vec_duplicate:VI_D
3407               (sign_extend:<VEL>
3408                 (match_operand:<VSUBEL> 4 "register_operand" "  r,  r,  r,  r"))))
3409           (match_operand:VI_D 2 "vector_merge_operand"       " vu,  0, vu,  0")))]
3410   "TARGET_VECTOR"
3411   "v<insn>.vx\t%0,%3,%4%p1"
3412   [(set_attr "type" "<int_binop_insn_type>")
3413    (set_attr "mode" "<MODE>")])
3414
3415 (define_insn "@pred_<sat_op><mode>"
3416   [(set (match_operand:VI 0 "register_operand"           "=vd, vd, vr, vr")
3417         (if_then_else:VI
3418           (unspec:<VM>
3419             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
3420              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
3421              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
3422              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
3423              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
3424              (reg:SI VL_REGNUM)
3425              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3426           (unspec:VI
3427             [(match_operand:VI 3 "register_operand"      " vr, vr, vr, vr")
3428              (match_operand:VI 4 "register_operand"      " vr, vr, vr, vr")] VSAT_OP)
3429           (match_operand:VI 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
3430   "TARGET_VECTOR"
3431   "v<sat_op>.vv\t%0,%3,%4%p1"
3432   [(set_attr "type" "<sat_insn_type>")
3433    (set_attr "mode" "<MODE>")])
3434
3435 ;; Handle GET_MODE_INNER (mode) = QImode, HImode, SImode.
3436 (define_insn "@pred_<sat_op><mode>_scalar"
3437   [(set (match_operand:VI_QHS 0 "register_operand"       "=vd, vr, vd, vr")
3438         (if_then_else:VI_QHS
3439           (unspec:<VM>
3440             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
3441              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
3442              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
3443              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
3444              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
3445              (reg:SI VL_REGNUM)
3446              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3447           (unspec:VI_QHS
3448             [(match_operand:VI_QHS 3 "register_operand"  " vr, vr, vr, vr")
3449              (match_operand:<VEL> 4 "reg_or_0_operand"   " rJ, rJ, rJ, rJ")] VSAT_ARITH_OP)
3450           (match_operand:VI_QHS 2 "vector_merge_operand" " vu,  0, vu,  0")))]
3451   "TARGET_VECTOR"
3452   "v<sat_op>.vx\t%0,%3,%z4%p1"
3453   [(set_attr "type" "<sat_insn_type>")
3454    (set_attr "mode" "<MODE>")])
3455
3456 (define_insn "@pred_<sat_op><mode>_scalar"
3457   [(set (match_operand:VI 0 "register_operand"             "=vd, vr, vd, vr")
3458         (if_then_else:VI
3459           (unspec:<VM>
3460             [(match_operand:<VM> 1 "vector_mask_operand"   " vm, vm,Wc1,Wc1")
3461              (match_operand 5 "vector_length_operand"      " rK, rK, rK, rK")
3462              (match_operand 6 "const_int_operand"          "  i,  i,  i,  i")
3463              (match_operand 7 "const_int_operand"          "  i,  i,  i,  i")
3464              (match_operand 8 "const_int_operand"          "  i,  i,  i,  i")
3465              (reg:SI VL_REGNUM)
3466              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3467           (unspec:VI
3468             [(match_operand:VI 3 "register_operand"        " vr, vr, vr, vr")
3469              (match_operand 4 "pmode_reg_or_uimm5_operand" " rK, rK, rK, rK")] VSAT_SHIFT_OP)
3470           (match_operand:VI 2 "vector_merge_operand"       " vu,  0, vu,  0")))]
3471   "TARGET_VECTOR"
3472   "v<sat_op>.v%o4\t%0,%3,%4%p1"
3473   [(set_attr "type" "<sat_insn_type>")
3474    (set_attr "mode" "<MODE>")])
3475
3476 ;; Handle GET_MODE_INNER (mode) = DImode. We need to split them since
3477 ;; we need to deal with SEW = 64 in RV32 system.
3478 (define_expand "@pred_<sat_op><mode>_scalar"
3479   [(set (match_operand:VI_D 0 "register_operand")
3480         (if_then_else:VI_D
3481           (unspec:<VM>
3482             [(match_operand:<VM> 1 "vector_mask_operand")
3483              (match_operand 5 "vector_length_operand")
3484              (match_operand 6 "const_int_operand")
3485              (match_operand 7 "const_int_operand")
3486              (match_operand 8 "const_int_operand")
3487              (reg:SI VL_REGNUM)
3488              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3489           (unspec:VI_D
3490             [(match_operand:VI_D 3 "register_operand")
3491              (match_operand:<VEL> 4 "reg_or_int_operand")] VSAT_ARITH_OP)
3492           (match_operand:VI_D 2 "vector_merge_operand")))]
3493   "TARGET_VECTOR"
3494 {
3495   if (riscv_vector::sew64_scalar_helper (
3496         operands,
3497         /* scalar op */&operands[4],
3498         /* vl */operands[5],
3499         <MODE>mode,
3500         <VM>mode,
3501         false,
3502         [] (rtx *operands, rtx boardcast_scalar) {
3503           emit_insn (gen_pred_<sat_op><mode> (operands[0], operands[1],
3504                operands[2], operands[3], boardcast_scalar, operands[5],
3505                operands[6], operands[7], operands[8]));
3506         }))
3507     DONE;
3508 })
3509
3510 (define_insn "*pred_<sat_op><mode>_scalar"
3511   [(set (match_operand:VI_D 0 "register_operand"         "=vd, vr, vd, vr")
3512         (if_then_else:VI_D
3513           (unspec:<VM>
3514             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
3515              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
3516              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
3517              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
3518              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
3519              (reg:SI VL_REGNUM)
3520              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3521           (unspec:VI_D
3522             [(match_operand:VI_D 3 "register_operand"    " vr, vr, vr, vr")
3523              (match_operand:<VEL> 4 "reg_or_0_operand"   " rJ, rJ, rJ, rJ")] VSAT_ARITH_OP)
3524           (match_operand:VI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")))]
3525   "TARGET_VECTOR"
3526   "v<sat_op>.vx\t%0,%3,%z4%p1"
3527   [(set_attr "type" "<sat_insn_type>")
3528    (set_attr "mode" "<MODE>")])
3529
3530 (define_insn "*pred_<sat_op><mode>_extended_scalar"
3531   [(set (match_operand:VI_D 0 "register_operand"            "=vd, vr, vd, vr")
3532         (if_then_else:VI_D
3533           (unspec:<VM>
3534             [(match_operand:<VM> 1 "vector_mask_operand"    " vm, vm,Wc1,Wc1")
3535              (match_operand 5 "vector_length_operand"       " rK, rK, rK, rK")
3536              (match_operand 6 "const_int_operand"           "  i,  i,  i,  i")
3537              (match_operand 7 "const_int_operand"           "  i,  i,  i,  i")
3538              (match_operand 8 "const_int_operand"           "  i,  i,  i,  i")
3539              (reg:SI VL_REGNUM)
3540              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3541           (unspec:VI_D
3542             [(match_operand:VI_D 3 "register_operand"       " vr, vr, vr, vr")
3543              (sign_extend:<VEL>
3544                (match_operand:<VSUBEL> 4 "reg_or_0_operand" " rJ, rJ, rJ, rJ"))] VSAT_ARITH_OP)
3545           (match_operand:VI_D 2 "vector_merge_operand"      " vu,  0, vu,  0")))]
3546   "TARGET_VECTOR"
3547   "v<sat_op>.vx\t%0,%3,%z4%p1"
3548   [(set_attr "type" "<sat_insn_type>")
3549    (set_attr "mode" "<MODE>")])
3550
3551 ;; CLIP
3552 (define_insn "@pred_narrow_clip<v_su><mode>"
3553   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"           "=vd,vd, vr, vr,vd, vr,  &vr,  &vr, vd, vr,  &vr,  &vr")
3554         (if_then_else:<V_DOUBLE_TRUNC>
3555           (unspec:<VM>
3556             [(match_operand:<VM> 1 "vector_mask_operand"               " vm,vm,Wc1,Wc1,vm,Wc1,vmWc1,vmWc1, vm,Wc1,vmWc1,vmWc1")
3557              (match_operand 5 "vector_length_operand"                  " rK,rK, rK, rK,rK, rK,   rK,   rK, rK, rK,   rK,   rK")
3558              (match_operand 6 "const_int_operand"                      "  i, i,  i,  i, i,  i,    i,    i,  i,  i,    i,    i")
3559              (match_operand 7 "const_int_operand"                      "  i, i,  i,  i, i,  i,    i,    i,  i,  i,    i,    i")
3560              (match_operand 8 "const_int_operand"                      "  i, i,  i,  i, i,  i,    i,    i,  i,  i,    i,    i")
3561              (reg:SI VL_REGNUM)
3562              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3563           (unspec:<V_DOUBLE_TRUNC>
3564             [(match_operand:VWEXTI 3 "register_operand"                " vr,vr, vr, vr, 0,  0,   vr,   vr,  0,  0,   vr,   vr")
3565              (match_operand:<V_DOUBLE_TRUNC> 4 "vector_shift_operand"  "  0, 0,  0,  0,vr, vr,   vr,   vr, vk, vk,   vk,   vk")] VNCLIP)
3566           (match_operand:<V_DOUBLE_TRUNC> 2 "vector_merge_operand"     "  0,vu,  0, vu,vu, vu,   vu,    0, vu, vu,   vu,    0")))]
3567   "TARGET_VECTOR"
3568   "vnclip<v_su>.w%o4\t%0,%3,%v4%p1"
3569   [(set_attr "type" "vnclip")
3570    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3571
3572 (define_insn "@pred_narrow_clip<v_su><mode>_scalar"
3573   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"           "=vd, vd, vr, vr,  &vr,  &vr")
3574         (if_then_else:<V_DOUBLE_TRUNC>
3575           (unspec:<VM>
3576             [(match_operand:<VM> 1 "vector_mask_operand"               " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
3577              (match_operand 5 "vector_length_operand"                  " rK, rK, rK, rK,   rK,   rK")
3578              (match_operand 6 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3579              (match_operand 7 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3580              (match_operand 8 "const_int_operand"                      "  i,  i,  i,  i,    i,    i")
3581              (reg:SI VL_REGNUM)
3582              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3583           (unspec:<V_DOUBLE_TRUNC>
3584             [(match_operand:VWEXTI 3 "register_operand"                "  0,  0,  0,  0,   vr,   vr")
3585              (match_operand 4 "pmode_reg_or_uimm5_operand"             " rK, rK, rK, rK,   rK,   rK")] VNCLIP)
3586           (match_operand:<V_DOUBLE_TRUNC> 2 "vector_merge_operand"     " vu,  0, vu,  0,   vu,    0")))]
3587   "TARGET_VECTOR"
3588   "vnclip<v_su>.w%o4\t%0,%3,%4%p1"
3589   [(set_attr "type" "vnclip")
3590    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
3591
3592 ;; -------------------------------------------------------------------------------
3593 ;; ---- Predicated integer comparison operations
3594 ;; -------------------------------------------------------------------------------
3595 ;; Includes:
3596 ;; - 11.8 Vector Integer Comparision Instructions
3597 ;; -------------------------------------------------------------------------------
3598
3599 (define_expand "@pred_cmp<mode>"
3600   [(set (match_operand:<VM> 0 "register_operand")
3601         (if_then_else:<VM>
3602           (unspec:<VM>
3603             [(match_operand:<VM> 1 "vector_mask_operand")
3604              (match_operand 6 "vector_length_operand")
3605              (match_operand 7 "const_int_operand")
3606              (match_operand 8 "const_int_operand")
3607              (reg:SI VL_REGNUM)
3608              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3609           (match_operator:<VM> 3 "comparison_except_ltge_operator"
3610              [(match_operand:VI 4 "register_operand")
3611               (match_operand:VI 5 "vector_arith_operand")])
3612           (match_operand:<VM> 2 "vector_merge_operand")))]
3613   "TARGET_VECTOR"
3614   {})
3615
3616 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
3617 (define_insn "*pred_cmp<mode>"
3618   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr,   vr,   vr")
3619         (if_then_else:<VM>
3620           (unspec:<VM>
3621             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1,vmWc1,vmWc1")
3622              (match_operand 6 "vector_length_operand"         "   rK,   rK,   rK,   rK")
3623              (match_operand 7 "const_int_operand"             "    i,    i,    i,    i")
3624              (match_operand 8 "const_int_operand"             "    i,    i,    i,    i")
3625              (reg:SI VL_REGNUM)
3626              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3627           (match_operator:<VM> 3 "comparison_except_ltge_operator"
3628              [(match_operand:VI 4 "register_operand"          "   vr,   vr,   vr,   vr")
3629               (match_operand:VI 5 "vector_arith_operand"      "   vr,   vr,   vi,   vi")])
3630           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0,   vu,    0")))]
3631   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3632   "vms%B3.v%o5\t%0,%4,%v5%p1"
3633   [(set_attr "type" "vicmp")
3634    (set_attr "mode" "<MODE>")])
3635
3636 ;; We use early-clobber for source LMUL > dest LMUL.
3637 (define_insn "*pred_cmp<mode>_narrow"
3638   [(set (match_operand:<VM> 0 "register_operand"              "=&vr,   &vr,  &vr,  &vr")
3639         (if_then_else:<VM>
3640           (unspec:<VM>
3641             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1,vmWc1,vmWc1")
3642              (match_operand 6 "vector_length_operand"         "   rK,   rK,   rK,   rK")
3643              (match_operand 7 "const_int_operand"             "    i,    i,    i,    i")
3644              (match_operand 8 "const_int_operand"             "    i,    i,    i,    i")
3645              (reg:SI VL_REGNUM)
3646              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3647           (match_operator:<VM> 3 "comparison_except_ltge_operator"
3648              [(match_operand:VI 4 "register_operand"          "   vr,   vr,   vr,   vr")
3649               (match_operand:VI 5 "vector_arith_operand"      "   vr,   vr,   vi,   vi")])
3650           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0,   vu,    0")))]
3651   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3652   "vms%B3.v%o5\t%0,%4,%v5%p1"
3653   [(set_attr "type" "vicmp")
3654    (set_attr "mode" "<MODE>")])
3655
3656 (define_expand "@pred_ltge<mode>"
3657   [(set (match_operand:<VM> 0 "register_operand")
3658         (if_then_else:<VM>
3659           (unspec:<VM>
3660             [(match_operand:<VM> 1 "vector_mask_operand")
3661              (match_operand 6 "vector_length_operand")
3662              (match_operand 7 "const_int_operand")
3663              (match_operand 8 "const_int_operand")
3664              (reg:SI VL_REGNUM)
3665              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3666           (match_operator:<VM> 3 "ltge_operator"
3667              [(match_operand:VI 4 "register_operand")
3668               (match_operand:VI 5 "vector_neg_arith_operand")])
3669           (match_operand:<VM> 2 "vector_merge_operand")))]
3670   "TARGET_VECTOR"
3671   {})
3672
3673 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
3674 (define_insn "*pred_ltge<mode>"
3675   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr,   vr,   vr")
3676         (if_then_else:<VM>
3677           (unspec:<VM>
3678             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1,vmWc1,vmWc1")
3679              (match_operand 6 "vector_length_operand"         "   rK,   rK,   rK,   rK")
3680              (match_operand 7 "const_int_operand"             "    i,    i,    i,    i")
3681              (match_operand 8 "const_int_operand"             "    i,    i,    i,    i")
3682              (reg:SI VL_REGNUM)
3683              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3684           (match_operator:<VM> 3 "ltge_operator"
3685              [(match_operand:VI 4 "register_operand"          "   vr,   vr,   vr,   vr")
3686               (match_operand:VI 5 "vector_neg_arith_operand"  "   vr,   vr,   vj,   vj")])
3687           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0,   vu,    0")))]
3688   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3689   "vms%B3.v%o5\t%0,%4,%v5%p1"
3690   [(set_attr "type" "vicmp")
3691    (set_attr "mode" "<MODE>")])
3692
3693 ;; We use early-clobber for source LMUL > dest LMUL.
3694 (define_insn "*pred_ltge<mode>_narrow"
3695   [(set (match_operand:<VM> 0 "register_operand"              "=&vr,   &vr,  &vr,  &vr")
3696         (if_then_else:<VM>
3697           (unspec:<VM>
3698             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1,vmWc1,vmWc1")
3699              (match_operand 6 "vector_length_operand"         "   rK,   rK,   rK,   rK")
3700              (match_operand 7 "const_int_operand"             "    i,    i,    i,    i")
3701              (match_operand 8 "const_int_operand"             "    i,    i,    i,    i")
3702              (reg:SI VL_REGNUM)
3703              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3704           (match_operator:<VM> 3 "ltge_operator"
3705              [(match_operand:VI 4 "register_operand"          "   vr,   vr,   vr,   vr")
3706               (match_operand:VI 5 "vector_neg_arith_operand"  "   vr,   vr,   vj,   vj")])
3707           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0,   vu,    0")))]
3708   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3709   "vms%B3.v%o5\t%0,%4,%v5%p1"
3710   [(set_attr "type" "vicmp")
3711    (set_attr "mode" "<MODE>")])
3712
3713 (define_expand "@pred_cmp<mode>_scalar"
3714   [(set (match_operand:<VM> 0 "register_operand")
3715         (if_then_else:<VM>
3716           (unspec:<VM>
3717             [(match_operand:<VM> 1 "vector_mask_operand")
3718              (match_operand 6 "vector_length_operand")
3719              (match_operand 7 "const_int_operand")
3720              (match_operand 8 "const_int_operand")
3721              (reg:SI VL_REGNUM)
3722              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3723           (match_operator:<VM> 3 "comparison_except_eqge_operator"
3724              [(match_operand:VI_QHS 4 "register_operand")
3725               (vec_duplicate:VI_QHS
3726                 (match_operand:<VEL> 5 "register_operand"))])
3727           (match_operand:<VM> 2 "vector_merge_operand")))]
3728   "TARGET_VECTOR"
3729   {})
3730
3731 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
3732 (define_insn "*pred_cmp<mode>_scalar"
3733   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr")
3734         (if_then_else:<VM>
3735           (unspec:<VM>
3736             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3737              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3738              (match_operand 7 "const_int_operand"             "    i,    i")
3739              (match_operand 8 "const_int_operand"             "    i,    i")
3740              (reg:SI VL_REGNUM)
3741              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3742           (match_operator:<VM> 3 "comparison_except_eqge_operator"
3743              [(match_operand:VI_QHS 4 "register_operand"      "   vr,   vr")
3744               (vec_duplicate:VI_QHS
3745                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))])
3746           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3747   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3748   "vms%B3.vx\t%0,%4,%5%p1"
3749   [(set_attr "type" "vicmp")
3750    (set_attr "mode" "<MODE>")])
3751
3752 ;; We use early-clobber for source LMUL > dest LMUL.
3753 (define_insn "*pred_cmp<mode>_scalar_narrow"
3754   [(set (match_operand:<VM> 0 "register_operand"               "=&vr,  &vr")
3755         (if_then_else:<VM>
3756           (unspec:<VM>
3757             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3758              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3759              (match_operand 7 "const_int_operand"             "    i,    i")
3760              (match_operand 8 "const_int_operand"             "    i,    i")
3761              (reg:SI VL_REGNUM)
3762              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3763           (match_operator:<VM> 3 "comparison_except_eqge_operator"
3764              [(match_operand:VI_QHS 4 "register_operand"      "   vr,   vr")
3765               (vec_duplicate:VI_QHS
3766                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))])
3767           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3768   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3769   "vms%B3.vx\t%0,%4,%5%p1"
3770   [(set_attr "type" "vicmp")
3771    (set_attr "mode" "<MODE>")])
3772
3773 (define_expand "@pred_eqne<mode>_scalar"
3774   [(set (match_operand:<VM> 0 "register_operand")
3775         (if_then_else:<VM>
3776           (unspec:<VM>
3777             [(match_operand:<VM> 1 "vector_mask_operand")
3778              (match_operand 6 "vector_length_operand")
3779              (match_operand 7 "const_int_operand")
3780              (match_operand 8 "const_int_operand")
3781              (reg:SI VL_REGNUM)
3782              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3783           (match_operator:<VM> 3 "equality_operator"
3784              [(vec_duplicate:VI_QHS
3785                 (match_operand:<VEL> 5 "register_operand"))
3786               (match_operand:VI_QHS 4 "register_operand")])
3787           (match_operand:<VM> 2 "vector_merge_operand")))]
3788   "TARGET_VECTOR"
3789   {})
3790
3791 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
3792 (define_insn "*pred_eqne<mode>_scalar"
3793   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr")
3794         (if_then_else:<VM>
3795           (unspec:<VM>
3796             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3797              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3798              (match_operand 7 "const_int_operand"             "    i,    i")
3799              (match_operand 8 "const_int_operand"             "    i,    i")
3800              (reg:SI VL_REGNUM)
3801              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3802           (match_operator:<VM> 3 "equality_operator"
3803              [(vec_duplicate:VI_QHS
3804                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))
3805               (match_operand:VI_QHS 4 "register_operand"      "   vr,   vr")])
3806           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3807   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3808   "vms%B3.vx\t%0,%4,%5%p1"
3809   [(set_attr "type" "vicmp")
3810    (set_attr "mode" "<MODE>")])
3811
3812 ;; We use early-clobber for source LMUL > dest LMUL.
3813 (define_insn "*pred_eqne<mode>_scalar_narrow"
3814   [(set (match_operand:<VM> 0 "register_operand"               "=&vr,  &vr")
3815         (if_then_else:<VM>
3816           (unspec:<VM>
3817             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3818              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3819              (match_operand 7 "const_int_operand"             "    i,    i")
3820              (match_operand 8 "const_int_operand"             "    i,    i")
3821              (reg:SI VL_REGNUM)
3822              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3823           (match_operator:<VM> 3 "equality_operator"
3824              [(vec_duplicate:VI_QHS
3825                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))
3826               (match_operand:VI_QHS 4 "register_operand"      "   vr,   vr")])
3827           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3828   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3829   "vms%B3.vx\t%0,%4,%5%p1"
3830   [(set_attr "type" "vicmp")
3831    (set_attr "mode" "<MODE>")])
3832
3833 ;; Handle GET_MODE_INNER (mode) = DImode. We need to split them since
3834 ;; we need to deal with SEW = 64 in RV32 system.
3835 (define_expand "@pred_cmp<mode>_scalar"
3836   [(set (match_operand:<VM> 0 "register_operand")
3837         (if_then_else:<VM>
3838           (unspec:<VM>
3839             [(match_operand:<VM> 1 "vector_mask_operand")
3840              (match_operand 6 "vector_length_operand")
3841              (match_operand 7 "const_int_operand")
3842              (match_operand 8 "const_int_operand")
3843              (reg:SI VL_REGNUM)
3844              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3845           (match_operator:<VM> 3 "comparison_except_eqge_operator"
3846              [(match_operand:VI_D 4 "register_operand")
3847               (vec_duplicate:VI_D
3848                 (match_operand:<VEL> 5 "reg_or_int_operand"))])
3849           (match_operand:<VM> 2 "vector_merge_operand")))]
3850   "TARGET_VECTOR"
3851 {
3852   enum rtx_code code = GET_CODE (operands[3]);
3853   if (riscv_vector::sew64_scalar_helper (
3854         operands,
3855         /* scalar op */&operands[5],
3856         /* vl */operands[6],
3857         <MODE>mode,
3858         <VM>mode,
3859         riscv_vector::has_vi_variant_p (code, operands[5]),
3860         code == LT || code == LTU ?
3861           [] (rtx *operands, rtx boardcast_scalar) {
3862             emit_insn (gen_pred_ltge<mode> (operands[0], operands[1],
3863                 operands[2], operands[3], operands[4], boardcast_scalar,
3864                 operands[6], operands[7], operands[8]));
3865           }
3866         :
3867           [] (rtx *operands, rtx boardcast_scalar) {
3868             emit_insn (gen_pred_cmp<mode> (operands[0], operands[1],
3869                 operands[2], operands[3], operands[4], boardcast_scalar,
3870                 operands[6], operands[7], operands[8]));
3871           }))
3872     DONE;
3873 })
3874
3875 (define_expand "@pred_eqne<mode>_scalar"
3876   [(set (match_operand:<VM> 0 "register_operand")
3877         (if_then_else:<VM>
3878           (unspec:<VM>
3879             [(match_operand:<VM> 1 "vector_mask_operand")
3880              (match_operand 6 "vector_length_operand")
3881              (match_operand 7 "const_int_operand")
3882              (match_operand 8 "const_int_operand")
3883              (reg:SI VL_REGNUM)
3884              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3885           (match_operator:<VM> 3 "equality_operator"
3886              [(vec_duplicate:VI_D
3887                 (match_operand:<VEL> 5 "reg_or_int_operand"))
3888               (match_operand:VI_D 4 "register_operand")])
3889           (match_operand:<VM> 2 "vector_merge_operand")))]
3890   "TARGET_VECTOR"
3891 {
3892   enum rtx_code code = GET_CODE (operands[3]);
3893   if (riscv_vector::sew64_scalar_helper (
3894         operands,
3895         /* scalar op */&operands[5],
3896         /* vl */operands[6],
3897         <MODE>mode,
3898         <VM>mode,
3899         riscv_vector::has_vi_variant_p (code, operands[5]),
3900         [] (rtx *operands, rtx boardcast_scalar) {
3901           emit_insn (gen_pred_cmp<mode> (operands[0], operands[1],
3902                 operands[2], operands[3], operands[4], boardcast_scalar,
3903                 operands[6], operands[7], operands[8]));
3904         }))
3905     DONE;
3906 })
3907
3908 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
3909 (define_insn "*pred_cmp<mode>_scalar"
3910   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr")
3911         (if_then_else:<VM>
3912           (unspec:<VM>
3913             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3914              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3915              (match_operand 7 "const_int_operand"             "    i,    i")
3916              (match_operand 8 "const_int_operand"             "    i,    i")
3917              (reg:SI VL_REGNUM)
3918              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3919           (match_operator:<VM> 3 "comparison_except_eqge_operator"
3920              [(match_operand:VI_D 4 "register_operand"        "   vr,   vr")
3921               (vec_duplicate:VI_D
3922                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))])
3923           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3924   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3925   "vms%B3.vx\t%0,%4,%5%p1"
3926   [(set_attr "type" "vicmp")
3927    (set_attr "mode" "<MODE>")])
3928
3929 ;; We use early-clobber for source LMUL > dest LMUL.
3930 (define_insn "*pred_cmp<mode>_scalar_narrow"
3931   [(set (match_operand:<VM> 0 "register_operand"               "=&vr,  &vr")
3932         (if_then_else:<VM>
3933           (unspec:<VM>
3934             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3935              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3936              (match_operand 7 "const_int_operand"             "    i,    i")
3937              (match_operand 8 "const_int_operand"             "    i,    i")
3938              (reg:SI VL_REGNUM)
3939              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3940           (match_operator:<VM> 3 "comparison_except_eqge_operator"
3941              [(match_operand:VI_D 4 "register_operand"        "   vr,   vr")
3942               (vec_duplicate:VI_D
3943                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))])
3944           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3945   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3946   "vms%B3.vx\t%0,%4,%5%p1"
3947   [(set_attr "type" "vicmp")
3948    (set_attr "mode" "<MODE>")])
3949
3950 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
3951 (define_insn "*pred_eqne<mode>_scalar"
3952   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr")
3953         (if_then_else:<VM>
3954           (unspec:<VM>
3955             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3956              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3957              (match_operand 7 "const_int_operand"             "    i,    i")
3958              (match_operand 8 "const_int_operand"             "    i,    i")
3959              (reg:SI VL_REGNUM)
3960              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3961           (match_operator:<VM> 3 "equality_operator"
3962              [(vec_duplicate:VI_D
3963                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))
3964               (match_operand:VI_D 4 "register_operand"        "   vr,   vr")])
3965           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3966   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3967   "vms%B3.vx\t%0,%4,%5%p1"
3968   [(set_attr "type" "vicmp")
3969    (set_attr "mode" "<MODE>")])
3970
3971 ;; We use early-clobber for source LMUL > dest LMUL.
3972 (define_insn "*pred_eqne<mode>_scalar_narrow"
3973   [(set (match_operand:<VM> 0 "register_operand"               "=&vr,  &vr")
3974         (if_then_else:<VM>
3975           (unspec:<VM>
3976             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
3977              (match_operand 6 "vector_length_operand"         "   rK,   rK")
3978              (match_operand 7 "const_int_operand"             "    i,    i")
3979              (match_operand 8 "const_int_operand"             "    i,    i")
3980              (reg:SI VL_REGNUM)
3981              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
3982           (match_operator:<VM> 3 "equality_operator"
3983              [(vec_duplicate:VI_D
3984                 (match_operand:<VEL> 5 "register_operand"     "    r,    r"))
3985               (match_operand:VI_D 4 "register_operand"        "   vr,   vr")])
3986           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
3987   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
3988   "vms%B3.vx\t%0,%4,%5%p1"
3989   [(set_attr "type" "vicmp")
3990    (set_attr "mode" "<MODE>")])
3991
3992 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
3993 (define_insn "*pred_cmp<mode>_extended_scalar"
3994   [(set (match_operand:<VM> 0 "register_operand"                 "=vr,   vr")
3995         (if_then_else:<VM>
3996           (unspec:<VM>
3997             [(match_operand:<VM> 1 "vector_mask_operand"       "vmWc1,vmWc1")
3998              (match_operand 6 "vector_length_operand"          "   rK,   rK")
3999              (match_operand 7 "const_int_operand"              "    i,    i")
4000              (match_operand 8 "const_int_operand"              "    i,    i")
4001              (reg:SI VL_REGNUM)
4002              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4003           (match_operator:<VM> 3 "comparison_except_eqge_operator"
4004              [(match_operand:VI_D 4 "register_operand"         "   vr,   vr")
4005               (vec_duplicate:VI_D
4006                 (sign_extend:<VEL>
4007                   (match_operand:<VSUBEL> 5 "register_operand" "    r,    r")))])
4008           (match_operand:<VM> 2 "vector_merge_operand"         "   vu,    0")))]
4009   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
4010   "vms%B3.vx\t%0,%4,%5%p1"
4011   [(set_attr "type" "vicmp")
4012    (set_attr "mode" "<MODE>")])
4013
4014 (define_insn "*pred_cmp<mode>_extended_scalar_narrow"
4015   [(set (match_operand:<VM> 0 "register_operand"                "=&vr,  &vr")
4016         (if_then_else:<VM>
4017           (unspec:<VM>
4018             [(match_operand:<VM> 1 "vector_mask_operand"       "vmWc1,vmWc1")
4019              (match_operand 6 "vector_length_operand"          "   rK,   rK")
4020              (match_operand 7 "const_int_operand"              "    i,    i")
4021              (match_operand 8 "const_int_operand"              "    i,    i")
4022              (reg:SI VL_REGNUM)
4023              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4024           (match_operator:<VM> 3 "comparison_except_eqge_operator"
4025              [(match_operand:VI_D 4 "register_operand"         "   vr,   vr")
4026               (vec_duplicate:VI_D
4027                 (sign_extend:<VEL>
4028                   (match_operand:<VSUBEL> 5 "register_operand" "    r,    r")))])
4029           (match_operand:<VM> 2 "vector_merge_operand"         "   vu,    0")))]
4030   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
4031   "vms%B3.vx\t%0,%4,%5%p1"
4032   [(set_attr "type" "vicmp")
4033    (set_attr "mode" "<MODE>")])
4034
4035 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
4036 (define_insn "*pred_eqne<mode>_extended_scalar"
4037   [(set (match_operand:<VM> 0 "register_operand"                 "=vr,   vr")
4038         (if_then_else:<VM>
4039           (unspec:<VM>
4040             [(match_operand:<VM> 1 "vector_mask_operand"       "vmWc1,vmWc1")
4041              (match_operand 6 "vector_length_operand"          "   rK,   rK")
4042              (match_operand 7 "const_int_operand"              "    i,    i")
4043              (match_operand 8 "const_int_operand"              "    i,    i")
4044              (reg:SI VL_REGNUM)
4045              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4046           (match_operator:<VM> 3 "equality_operator"
4047              [(vec_duplicate:VI_D
4048                 (sign_extend:<VEL>
4049                   (match_operand:<VSUBEL> 5 "register_operand" "    r,    r")))
4050               (match_operand:VI_D 4 "register_operand"         "   vr,   vr")])
4051           (match_operand:<VM> 2 "vector_merge_operand"         "   vu,    0")))]
4052   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
4053   "vms%B3.vx\t%0,%4,%5%p1"
4054   [(set_attr "type" "vicmp")
4055    (set_attr "mode" "<MODE>")])
4056
4057 (define_insn "*pred_eqne<mode>_extended_scalar_narrow"
4058   [(set (match_operand:<VM> 0 "register_operand"                "=&vr,  &vr")
4059         (if_then_else:<VM>
4060           (unspec:<VM>
4061             [(match_operand:<VM> 1 "vector_mask_operand"       "vmWc1,vmWc1")
4062              (match_operand 6 "vector_length_operand"          "   rK,   rK")
4063              (match_operand 7 "const_int_operand"              "    i,    i")
4064              (match_operand 8 "const_int_operand"              "    i,    i")
4065              (reg:SI VL_REGNUM)
4066              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4067           (match_operator:<VM> 3 "equality_operator"
4068              [(vec_duplicate:VI_D
4069                 (sign_extend:<VEL>
4070                   (match_operand:<VSUBEL> 5 "register_operand" "    r,    r")))
4071               (match_operand:VI_D 4 "register_operand"         "   vr,   vr")])
4072           (match_operand:<VM> 2 "vector_merge_operand"         "   vu,    0")))]
4073   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
4074   "vms%B3.vx\t%0,%4,%5%p1"
4075   [(set_attr "type" "vicmp")
4076    (set_attr "mode" "<MODE>")])
4077
4078 ;; GE, vmsge.vx/vmsgeu.vx
4079 ;;
4080 ;; unmasked va >= x
4081 ;;  - pseudoinstruction: vmsge{u}.vx vd, va, x
4082 ;;  - expansion: vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
4083 ;;
4084 ;; masked va >= x, vd != v0
4085 ;;  - pseudoinstruction: vmsge{u}.vx vd, va, x, v0.t
4086 ;;  - expansion: vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
4087 ;;
4088 ;; masked va >= x, vd == v0
4089 ;;  - pseudoinstruction: vmsge{u}.vx vd, va, x, v0.t, vt
4090 ;;  - expansion: vmslt{u}.vx vt, va, x;  vmandn.mm vd, vd, vt
4091 (define_expand "@pred_ge<mode>_scalar"
4092   [(set (match_operand:<VM> 0 "register_operand")
4093         (if_then_else:<VM>
4094           (unspec:<VM>
4095             [(match_operand:<VM> 1 "vector_mask_operand")
4096              (match_operand 6 "vector_length_operand")
4097              (match_operand 7 "const_int_operand")
4098              (match_operand 8 "const_int_operand")
4099              (reg:SI VL_REGNUM)
4100              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4101           (match_operator:<VM> 3 "ge_operator"
4102              [(match_operand:VI 4 "register_operand")
4103               (vec_duplicate:VI
4104                 (match_operand:<VEL> 5 "reg_or_int_operand"))])
4105           (match_operand:<VM> 2 "vector_merge_operand")))]
4106   "TARGET_VECTOR"
4107 {
4108   enum rtx_code code = GET_CODE (operands[3]);
4109   rtx undef = RVV_VUNDEF (<VM>mode);
4110   if (code == GEU && rtx_equal_p (operands[5], const0_rtx))
4111     {
4112       /* If vmsgeu with 0 immediate, expand it to vmset.  */
4113       if (satisfies_constraint_Wc1 (operands[1]))
4114         emit_insn (
4115           gen_pred_mov (<VM>mode, operands[0], CONSTM1_RTX (<VM>mode), undef,
4116                         CONSTM1_RTX (<VM>mode), operands[6], operands[8]));
4117       else
4118         {
4119           /* If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff.
4120            */
4121           if (rtx_equal_p (operands[1], operands[2]))
4122             emit_move_insn (operands[0], operands[1]);
4123           else if (register_operand (operands[2], <VM>mode))
4124             emit_insn (gen_pred (IOR, <VM>mode, operands[0],
4125                                  CONSTM1_RTX (<VM>mode), undef, operands[1],
4126                                  operands[2], operands[6], operands[8]));
4127           else
4128             emit_insn (gen_pred (IOR, <VM>mode, operands[0],
4129                                  CONSTM1_RTX (<VM>mode), undef, operands[1],
4130                                  operands[1], operands[6], operands[8]));
4131         }
4132     }
4133   else if (riscv_vector::neg_simm5_p (operands[5]))
4134     emit_insn (
4135       gen_pred_ltge<mode> (operands[0], operands[1], operands[2], operands[3],
4136                            operands[4],
4137                            gen_const_vec_duplicate (<MODE>mode, operands[5]),
4138                            operands[6], operands[7], operands[8]));
4139   else
4140     {
4141       if (code == GE)
4142         operands[3] = gen_rtx_fmt_ee (LT, <VM>mode, XEXP (operands[3], 0),
4143                                       XEXP (operands[3], 1));
4144       else
4145         operands[3] = gen_rtx_fmt_ee (LTU, <VM>mode, XEXP (operands[3], 0),
4146                                       XEXP (operands[3], 1));
4147       if (GET_MODE_BITSIZE (<VEL>mode) <= GET_MODE_BITSIZE (Pmode))
4148         operands[5] = force_reg (<VEL>mode, operands[5]);
4149
4150       if (satisfies_constraint_Wc1 (operands[1]))
4151         {
4152           /* unmasked va >= x
4153             - pseudoinstruction: vmsge{u}.vx vd, va, x
4154             - expansion: vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd.  */
4155           emit_insn (
4156             gen_pred_cmp<mode>_scalar (operands[0], operands[1], operands[2],
4157                                         operands[3], operands[4], operands[5],
4158                                         operands[6], operands[7], operands[8]));
4159           emit_insn (gen_pred_nand<vm> (operands[0], CONSTM1_RTX (<VM>mode),
4160                                         undef, operands[0], operands[0],
4161                                         operands[6], operands[8]));
4162         }
4163       else
4164         {
4165           if (rtx_equal_p (operands[1], operands[2]))
4166             {
4167               /* masked va >= x, vd == v0
4168                 - pseudoinstruction: vmsge{u}.vx vd, va, x, v0.t, vt
4169                 - expansion: vmslt{u}.vx vt, va, x;  vmandn.mm vd, vd, vt.  */
4170               rtx reg = gen_reg_rtx (<VM>mode);
4171               emit_insn (gen_pred_cmp<mode>_scalar (
4172                 reg, CONSTM1_RTX (<VM>mode), undef, operands[3], operands[4],
4173                 operands[5], operands[6], operands[7], operands[8]));
4174               emit_insn (
4175                 gen_pred_andnot<vm> (operands[0], CONSTM1_RTX (<VM>mode), undef,
4176                                    operands[1], reg, operands[6], operands[8]));
4177             }
4178           else
4179             {
4180               /* masked va >= x, vd != v0
4181                 - pseudoinstruction: vmsge{u}.vx vd, va, x, v0.t
4182                 - expansion: vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0.
4183               */
4184               emit_insn (gen_pred_cmp<mode>_scalar (
4185                 operands[0], operands[1], operands[2], operands[3], operands[4],
4186                 operands[5], operands[6], operands[7], operands[8]));
4187               emit_insn (gen_pred (XOR, <VM>mode, operands[0],
4188                                    CONSTM1_RTX (<VM>mode), undef, operands[0],
4189                                    operands[1], operands[6], operands[8]));
4190             }
4191         }
4192     }
4193   DONE;
4194 })
4195
4196 ;; -------------------------------------------------------------------------------
4197 ;; ---- Predicated integer ternary operations
4198 ;; -------------------------------------------------------------------------------
4199 ;; Includes:
4200 ;; - 11.13 Vector Single-Width Integer Multiply-Add Instructions
4201 ;; -------------------------------------------------------------------------------
4202
4203 (define_expand "@pred_mul_plus<mode>"
4204   [(set (match_operand:VI 0 "register_operand")
4205         (if_then_else:VI
4206           (unspec:<VM>
4207             [(match_operand:<VM> 1 "vector_mask_operand")
4208              (match_operand 6 "vector_length_operand")
4209              (match_operand 7 "const_int_operand")
4210              (match_operand 8 "const_int_operand")
4211              (match_operand 9 "const_int_operand")
4212              (reg:SI VL_REGNUM)
4213              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4214           (plus:VI
4215             (mult:VI
4216               (match_operand:VI 2 "register_operand")
4217               (match_operand:VI 3 "register_operand"))
4218             (match_operand:VI 4 "register_operand"))
4219           (match_operand:VI 5 "register_operand")))]
4220   "TARGET_VECTOR"
4221 {
4222   /* Swap the multiplication operands if the fallback value is the
4223      second of the two.  */
4224   if (rtx_equal_p (operands[3], operands[5]))
4225     std::swap (operands[2], operands[3]);
4226 })
4227
4228 (define_insn "*pred_madd<mode>"
4229   [(set (match_operand:VI 0 "register_operand"           "=vd,?&vd, vr,?&vr")
4230         (if_then_else:VI
4231           (unspec:<VM>
4232             [(match_operand:<VM> 1 "vector_mask_operand" " vm,  vm,Wc1, Wc1")
4233              (match_operand 5 "vector_length_operand"    " rK,  rK, rK,  rK")
4234              (match_operand 6 "const_int_operand"        "  i,   i,  i,   i")
4235              (match_operand 7 "const_int_operand"        "  i,   i,  i,   i")
4236              (match_operand 8 "const_int_operand"        "  i,   i,  i,   i")
4237              (reg:SI VL_REGNUM)
4238              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4239           (plus:VI
4240             (mult:VI
4241               (match_operand:VI 2 "register_operand"     "  0,  vr,  0,  vr")
4242               (match_operand:VI 3 "register_operand"     " vr,  vr, vr,  vr"))
4243             (match_operand:VI 4 "register_operand"       " vr,  vr, vr,  vr"))
4244           (match_dup 2)))]
4245   "TARGET_VECTOR"
4246   "@
4247    vmadd.vv\t%0,%3,%4%p1
4248    vmv.v.v\t%0,%2\;vmadd.vv\t%0,%3,%4%p1
4249    vmadd.vv\t%0,%3,%4%p1
4250    vmv.v.v\t%0,%2\;vmadd.vv\t%0,%3,%4%p1"
4251   [(set_attr "type" "vimuladd")
4252    (set_attr "mode" "<MODE>")
4253    (set_attr "merge_op_idx" "4")
4254    (set_attr "vl_op_idx" "5")
4255    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4256    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4257    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4258
4259 (define_insn "*pred_macc<mode>"
4260   [(set (match_operand:VI 0 "register_operand"           "=vd,?&vd, vr,?&vr")
4261         (if_then_else:VI
4262           (unspec:<VM>
4263             [(match_operand:<VM> 1 "vector_mask_operand" " vm,  vm,Wc1, Wc1")
4264              (match_operand 5 "vector_length_operand"    " rK,  rK, rK,  rK")
4265              (match_operand 6 "const_int_operand"        "  i,   i,  i,   i")
4266              (match_operand 7 "const_int_operand"        "  i,   i,  i,   i")
4267              (match_operand 8 "const_int_operand"        "  i,   i,  i,   i")
4268              (reg:SI VL_REGNUM)
4269              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4270           (plus:VI
4271             (mult:VI
4272               (match_operand:VI 2 "register_operand"     " vr,  vr, vr,  vr")
4273               (match_operand:VI 3 "register_operand"     " vr,  vr, vr,  vr"))
4274             (match_operand:VI 4 "register_operand"       "  0,  vr,  0,  vr"))
4275           (match_dup 4)))]
4276   "TARGET_VECTOR"
4277   "@
4278    vmacc.vv\t%0,%2,%3%p1
4279    vmv.v.v\t%0,%4\;vmacc.vv\t%0,%2,%3%p1
4280    vmacc.vv\t%0,%2,%3%p1
4281    vmv.v.v\t%0,%4\;vmacc.vv\t%0,%2,%3%p1"
4282   [(set_attr "type" "vimuladd")
4283    (set_attr "mode" "<MODE>")
4284    (set_attr "merge_op_idx" "2")
4285    (set_attr "vl_op_idx" "5")
4286    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4287    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4288    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4289
4290 (define_insn_and_rewrite "*pred_mul_plus<mode>"
4291   [(set (match_operand:VI 0 "register_operand"            "=&vr,?&vr, ?&vr, ?&vr,  ?&vr")
4292         (if_then_else:VI
4293           (unspec:<VM>
4294             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1,vmWc1")
4295              (match_operand 6 "vector_length_operand"    "   rK,   rK,   rK,   rK,   rK")
4296              (match_operand 7 "const_int_operand"        "    i,    i,    i,    i,    i")
4297              (match_operand 8 "const_int_operand"        "    i,    i,    i,    i,    i")
4298              (match_operand 9 "const_int_operand"        "    i,    i,    i,    i,    i")
4299              (reg:SI VL_REGNUM)
4300              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4301           (plus:VI
4302             (mult:VI
4303               (match_operand:VI 2 "register_operand"     "   vr,   vr,   vi,   vr,   vr")
4304               (match_operand:VI 3 "register_operand"     "   vr,   vr,   vr,   vi,   vr"))
4305             (match_operand:VI 4 "vector_arith_operand"   "   vr,   vi,   vr,   vr,   vr"))
4306           (match_operand:VI 5 "register_operand"         "    0,   vr,   vr,   vr,   vr")))]
4307   "TARGET_VECTOR
4308    && !rtx_equal_p (operands[2], operands[5])
4309    && !rtx_equal_p (operands[3], operands[5])
4310    && !rtx_equal_p (operands[4], operands[5])"
4311   "@
4312    vmv.v.v\t%0,%4\;vmacc.vv\t%0,%2,%3%p1
4313    #
4314    #
4315    #
4316    #"
4317   "&& reload_completed
4318    && !rtx_equal_p (operands[0], operands[5])"
4319   {
4320     if (satisfies_constraint_vi (operands[3]))
4321       std::swap (operands[2], operands[3]);
4322
4323     if (satisfies_constraint_vi (operands[2]))
4324       {
4325         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4326                         operands[5], operands[2], operands[1], operands[6],
4327                         operands[7], operands[9]));
4328         operands[5] = operands[2] = operands[0];
4329       }
4330     else
4331       {
4332         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4333                         operands[5], operands[4], operands[1], operands[6],
4334                         operands[7], operands[9]));
4335         operands[5] = operands[4] = operands[0];
4336       }
4337   }
4338   [(set_attr "type" "vimuladd")
4339    (set_attr "mode" "<MODE>")])
4340
4341 (define_expand "@pred_mul_plus<mode>_scalar"
4342   [(set (match_operand:VI_QHS 0 "register_operand")
4343         (if_then_else:VI_QHS
4344           (unspec:<VM>
4345             [(match_operand:<VM> 1 "vector_mask_operand")
4346              (match_operand 6 "vector_length_operand")
4347              (match_operand 7 "const_int_operand")
4348              (match_operand 8 "const_int_operand")
4349              (match_operand 9 "const_int_operand")
4350              (reg:SI VL_REGNUM)
4351              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4352           (plus:VI_QHS
4353             (mult:VI_QHS
4354               (vec_duplicate:VI_QHS
4355                 (match_operand:<VEL> 2 "reg_or_int_operand"))
4356               (match_operand:VI_QHS 3 "register_operand"))
4357             (match_operand:VI_QHS 4 "register_operand"))
4358           (match_operand:VI_QHS 5 "register_operand")))]
4359   "TARGET_VECTOR"
4360 {
4361   operands[2] = force_reg (<VEL>mode, operands[2]);
4362 })
4363
4364 (define_insn "*pred_madd<mode>_scalar"
4365   [(set (match_operand:VI 0 "register_operand"            "=vd,?&vd, vr,?&vr")
4366         (if_then_else:VI
4367           (unspec:<VM>
4368             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,  vm,Wc1, Wc1")
4369              (match_operand 5 "vector_length_operand"     " rK,  rK, rK,  rK")
4370              (match_operand 6 "const_int_operand"         "  i,   i,  i,   i")
4371              (match_operand 7 "const_int_operand"         "  i,   i,  i,   i")
4372              (match_operand 8 "const_int_operand"         "  i,   i,  i,   i")
4373              (reg:SI VL_REGNUM)
4374              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4375           (plus:VI
4376             (mult:VI
4377               (vec_duplicate:VI
4378                 (match_operand:<VEL> 2 "register_operand" "  r,   r,  r,   r"))
4379               (match_operand:VI 3 "register_operand"      "  0,  vr,  0,  vr"))
4380             (match_operand:VI 4 "register_operand"        " vr,  vr, vr,  vr"))
4381           (match_dup 3)))]
4382   "TARGET_VECTOR"
4383   "@
4384    vmadd.vx\t%0,%2,%4%p1
4385    vmv.v.v\t%0,%3\;vmadd.vx\t%0,%2,%4%p1
4386    vmadd.vx\t%0,%2,%4%p1
4387    vmv.v.v\t%0,%3\;vmadd.vx\t%0,%2,%4%p1"
4388   [(set_attr "type" "vimuladd")
4389    (set_attr "mode" "<MODE>")
4390    (set_attr "merge_op_idx" "4")
4391    (set_attr "vl_op_idx" "5")
4392    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4393    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4394    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4395
4396 (define_insn "*pred_macc<mode>_scalar"
4397   [(set (match_operand:VI 0 "register_operand"            "=vd,?&vd, vr,?&vr")
4398         (if_then_else:VI
4399           (unspec:<VM>
4400             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,  vm,Wc1, Wc1")
4401              (match_operand 5 "vector_length_operand"     " rK,  rK, rK,  rK")
4402              (match_operand 6 "const_int_operand"         "  i,   i,  i,   i")
4403              (match_operand 7 "const_int_operand"         "  i,   i,  i,   i")
4404              (match_operand 8 "const_int_operand"         "  i,   i,  i,   i")
4405              (reg:SI VL_REGNUM)
4406              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4407           (plus:VI
4408             (mult:VI
4409               (vec_duplicate:VI
4410                 (match_operand:<VEL> 2 "register_operand" "  r,   r,  r,   r"))
4411               (match_operand:VI 3 "register_operand"      " vr,  vr, vr,  vr"))
4412             (match_operand:VI 4 "register_operand"        "  0,  vr,  0,  vr"))
4413           (match_dup 4)))]
4414   "TARGET_VECTOR"
4415   "@
4416    vmacc.vx\t%0,%2,%3%p1
4417    vmv.v.v\t%0,%4\;vmacc.vx\t%0,%2,%3%p1
4418    vmacc.vx\t%0,%2,%3%p1
4419    vmv.v.v\t%0,%4\;vmacc.vx\t%0,%2,%3%p1"
4420   [(set_attr "type" "vimuladd")
4421    (set_attr "mode" "<MODE>")
4422    (set_attr "merge_op_idx" "2")
4423    (set_attr "vl_op_idx" "5")
4424    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4425    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4426    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4427
4428 (define_insn_and_rewrite "*pred_mul_plus<mode>_scalar"
4429   [(set (match_operand:VI 0 "register_operand"            "=&vr, ?&vr, ?&vr, ?&vr")
4430         (if_then_else:VI
4431           (unspec:<VM>
4432             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1")
4433              (match_operand 6 "vector_length_operand"    "   rK,   rK,   rK,   rK")
4434              (match_operand 7 "const_int_operand"        "    i,    i,    i,    i")
4435              (match_operand 8 "const_int_operand"        "    i,    i,    i,    i")
4436              (match_operand 9 "const_int_operand"        "    i,    i,    i,    i")
4437              (reg:SI VL_REGNUM)
4438              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4439           (plus:VI
4440             (mult:VI
4441               (vec_duplicate:VI
4442                 (match_operand:<VEL> 2 "register_operand" "    r,    r,    r,    r"))
4443               (match_operand:VI 3 "register_operand"      "   vr,   vr,   vi,   vr"))
4444             (match_operand:VI 4 "vector_arith_operand"    "   vr,   vi,   vr,   vr"))
4445           (match_operand:VI 5 "register_operand"          "    0,   vr,   vr,   vr")))]
4446   "TARGET_VECTOR
4447    && !rtx_equal_p (operands[3], operands[5])
4448    && !rtx_equal_p (operands[4], operands[5])"
4449   "@
4450    vmv.v.v\t%0,%4\;vmacc.vx\t%0,%2,%3%p1
4451    #
4452    #
4453    #"
4454   "&& reload_completed
4455    && !rtx_equal_p (operands[0], operands[5])"
4456   {
4457     if (satisfies_constraint_vi (operands[3]))
4458       {
4459         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4460                         operands[5], operands[3], operands[1], operands[6],
4461                         operands[7], operands[9]));
4462         operands[5] = operands[3] = operands[0];
4463       }
4464     else
4465       {
4466         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4467                         operands[5], operands[4], operands[1], operands[6],
4468                         operands[7], operands[9]));
4469         operands[5] = operands[4] = operands[0];
4470       }
4471   }
4472   [(set_attr "type" "vimuladd")
4473    (set_attr "mode" "<MODE>")])
4474
4475 (define_expand "@pred_mul_plus<mode>_scalar"
4476   [(set (match_operand:VI_D 0 "register_operand")
4477         (if_then_else:VI_D
4478           (unspec:<VM>
4479             [(match_operand:<VM> 1 "vector_mask_operand")
4480              (match_operand 6 "vector_length_operand")
4481              (match_operand 7 "const_int_operand")
4482              (match_operand 8 "const_int_operand")
4483              (match_operand 9 "const_int_operand")
4484              (reg:SI VL_REGNUM)
4485              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4486           (plus:VI_D
4487             (mult:VI_D
4488               (vec_duplicate:VI_D
4489                 (match_operand:<VEL> 2 "reg_or_int_operand"))
4490               (match_operand:VI_D 3 "register_operand"))
4491             (match_operand:VI_D 4 "register_operand"))
4492           (match_operand:VI_D 5 "register_operand")))]
4493   "TARGET_VECTOR"
4494 {
4495   if (riscv_vector::sew64_scalar_helper (
4496         operands,
4497         /* scalar op */&operands[2],
4498         /* vl */operands[6],
4499         <MODE>mode,
4500         <VM>mode,
4501         false,
4502         [] (rtx *operands, rtx boardcast_scalar) {
4503           emit_insn (gen_pred_mul_plus<mode> (operands[0], operands[1],
4504                boardcast_scalar, operands[3], operands[4], operands[5],
4505                operands[6], operands[7], operands[8], operands[9]));
4506         }))
4507     DONE;
4508 })
4509
4510 (define_insn "*pred_madd<mode>_extended_scalar"
4511   [(set (match_operand:VI_D 0 "register_operand"               "=vd,?&vd, vr,?&vr")
4512         (if_then_else:VI_D
4513           (unspec:<VM>
4514             [(match_operand:<VM> 1 "vector_mask_operand"       " vm,  vm,Wc1, Wc1")
4515              (match_operand 5 "vector_length_operand"          " rK,  rK, rK,  rK")
4516              (match_operand 6 "const_int_operand"              "  i,   i,  i,   i")
4517              (match_operand 7 "const_int_operand"              "  i,   i,  i,   i")
4518              (match_operand 8 "const_int_operand"              "  i,   i,  i,   i")
4519              (reg:SI VL_REGNUM)
4520              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4521           (plus:VI_D
4522             (mult:VI_D
4523               (vec_duplicate:VI_D
4524                 (sign_extend:<VEL>
4525                   (match_operand:<VSUBEL> 2 "register_operand" "  r,   r,  r,   r")))
4526               (match_operand:VI_D 3 "register_operand"         "  0,  vr,  0,  vr"))
4527             (match_operand:VI_D 4 "register_operand"           " vr,  vr, vr,  vr"))
4528           (match_dup 3)))]
4529   "TARGET_VECTOR"
4530   "@
4531    vmadd.vx\t%0,%2,%4%p1
4532    vmv.v.v\t%0,%2\;vmadd.vx\t%0,%2,%4%p1
4533    vmadd.vx\t%0,%2,%4%p1
4534    vmv.v.v\t%0,%2\;vmadd.vx\t%0,%2,%4%p1"
4535   [(set_attr "type" "vimuladd")
4536    (set_attr "mode" "<MODE>")
4537    (set_attr "merge_op_idx" "4")
4538    (set_attr "vl_op_idx" "5")
4539    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4540    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4541    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4542
4543 (define_insn "*pred_macc<mode>_extended_scalar"
4544   [(set (match_operand:VI_D 0 "register_operand"               "=vd,?&vd, vr,?&vr")
4545         (if_then_else:VI_D
4546           (unspec:<VM>
4547             [(match_operand:<VM> 1 "vector_mask_operand"       " vm,  vm,Wc1, Wc1")
4548              (match_operand 5 "vector_length_operand"          " rK,  rK, rK,  rK")
4549              (match_operand 6 "const_int_operand"              "  i,   i,  i,   i")
4550              (match_operand 7 "const_int_operand"              "  i,   i,  i,   i")
4551              (match_operand 8 "const_int_operand"              "  i,   i,  i,   i")
4552              (reg:SI VL_REGNUM)
4553              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4554           (plus:VI_D
4555             (mult:VI_D
4556               (vec_duplicate:VI_D
4557                 (sign_extend:<VEL>
4558                   (match_operand:<VSUBEL> 2 "register_operand" "  r,   r,  r,   r")))
4559               (match_operand:VI_D 3 "register_operand"         " vr,  vr, vr,  vr"))
4560             (match_operand:VI_D 4 "register_operand"           "  0,  vr,  0,  vr"))
4561           (match_dup 4)))]
4562   "TARGET_VECTOR"
4563   "@
4564    vmacc.vx\t%0,%2,%3%p1
4565    vmv.v.v\t%0,%4\;vmacc.vx\t%0,%2,%3%p1
4566    vmacc.vx\t%0,%2,%3%p1
4567    vmv.v.v\t%0,%4\;vmacc.vx\t%0,%2,%3%p1"
4568   [(set_attr "type" "vimuladd")
4569    (set_attr "mode" "<MODE>")
4570    (set_attr "merge_op_idx" "2")
4571    (set_attr "vl_op_idx" "5")
4572    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4573    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4574    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4575
4576 (define_insn_and_rewrite "*pred_mul_plus<mode>_extended_scalar"
4577   [(set (match_operand:VI_D 0 "register_operand"                "=&vr, ?&vr, ?&vr, ?&vr")
4578         (if_then_else:VI_D
4579           (unspec:<VM>
4580             [(match_operand:<VM> 1 "vector_mask_operand"       "vmWc1,vmWc1,vmWc1,vmWc1")
4581              (match_operand 6 "vector_length_operand"          "   rK,   rK,   rK,   rK")
4582              (match_operand 7 "const_int_operand"              "    i,    i,    i,    i")
4583              (match_operand 8 "const_int_operand"              "    i,    i,    i,    i")
4584              (match_operand 9 "const_int_operand"              "    i,    i,    i,    i")
4585              (reg:SI VL_REGNUM)
4586              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4587           (plus:VI_D
4588             (mult:VI_D
4589               (vec_duplicate:VI_D
4590                 (sign_extend:<VEL>
4591                   (match_operand:<VSUBEL> 2 "register_operand" "    r,    r,    r,    r")))
4592               (match_operand:VI_D 3 "register_operand"         "   vr,   vr,   vr,   vr"))
4593             (match_operand:VI_D 4 "vector_arith_operand"       "   vr,   vr,   vr,   vr"))
4594           (match_operand:VI_D 5 "register_operand"             "    0,   vr,   vr,   vr")))]
4595   "TARGET_VECTOR
4596    && !rtx_equal_p (operands[3], operands[5])
4597    && !rtx_equal_p (operands[4], operands[5])"
4598   "@
4599    vmv.v.v\t%0,%4\;vmacc.vx\t%0,%2,%3%p1
4600    #
4601    #
4602    #"
4603   "&& reload_completed
4604    && !rtx_equal_p (operands[0], operands[5])"
4605   {
4606     if (satisfies_constraint_vi (operands[3]))
4607       {
4608         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4609                         operands[5], operands[3], operands[1], operands[6],
4610                         operands[7], operands[9]));
4611         operands[5] = operands[3] = operands[0];
4612       }
4613     else
4614       {
4615         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4616                         operands[5], operands[4], operands[1], operands[6],
4617                         operands[7], operands[9]));
4618         operands[5] = operands[4] = operands[0];
4619       }
4620   }
4621   [(set_attr "type" "vimuladd")
4622    (set_attr "mode" "<MODE>")])
4623
4624 (define_expand "@pred_minus_mul<mode>"
4625   [(set (match_operand:VI 0 "register_operand")
4626         (if_then_else:VI
4627           (unspec:<VM>
4628             [(match_operand:<VM> 1 "vector_mask_operand")
4629              (match_operand 6 "vector_length_operand")
4630              (match_operand 7 "const_int_operand")
4631              (match_operand 8 "const_int_operand")
4632              (match_operand 9 "const_int_operand")
4633              (reg:SI VL_REGNUM)
4634              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4635           (minus:VI
4636             (match_operand:VI 4 "register_operand")
4637             (mult:VI
4638               (match_operand:VI 2 "register_operand")
4639               (match_operand:VI 3 "register_operand")))
4640           (match_operand:VI 5 "register_operand")))]
4641   "TARGET_VECTOR"
4642 {
4643   /* Swap the multiplication operands if the fallback value is the
4644      second of the two.  */
4645   if (rtx_equal_p (operands[3], operands[5]))
4646     std::swap (operands[2], operands[3]);
4647 })
4648
4649 (define_insn "*pred_nmsub<mode>"
4650   [(set (match_operand:VI 0 "register_operand"           "=vd,?&vd, vr,?&vr")
4651         (if_then_else:VI
4652           (unspec:<VM>
4653             [(match_operand:<VM> 1 "vector_mask_operand" " vm,  vm,Wc1, Wc1")
4654              (match_operand 5 "vector_length_operand"    " rK,  rK, rK,  rK")
4655              (match_operand 6 "const_int_operand"        "  i,   i,  i,   i")
4656              (match_operand 7 "const_int_operand"        "  i,   i,  i,   i")
4657              (match_operand 8 "const_int_operand"        "  i,   i,  i,   i")
4658              (reg:SI VL_REGNUM)
4659              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4660           (minus:VI
4661             (match_operand:VI 4 "register_operand"       " vr,  vr, vr,  vr")
4662             (mult:VI
4663               (match_operand:VI 2 "register_operand"     "  0,  vr,  0,  vr")
4664               (match_operand:VI 3 "register_operand"     " vr,  vr, vr,  vr")))
4665           (match_dup 2)))]
4666   "TARGET_VECTOR"
4667   "@
4668    vnmsub.vv\t%0,%3,%4%p1
4669    vmv.v.v\t%0,%2\;vnmsub.vv\t%0,%3,%4%p1
4670    vnmsub.vv\t%0,%3,%4%p1
4671    vmv.v.v\t%0,%2\;vnmsub.vv\t%0,%3,%4%p1"
4672   [(set_attr "type" "vimuladd")
4673    (set_attr "mode" "<MODE>")
4674    (set_attr "merge_op_idx" "4")
4675    (set_attr "vl_op_idx" "5")
4676    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4677    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4678    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4679
4680 (define_insn "*pred_nmsac<mode>"
4681   [(set (match_operand:VI 0 "register_operand"           "=vd,?&vd, vr,?&vr")
4682         (if_then_else:VI
4683           (unspec:<VM>
4684             [(match_operand:<VM> 1 "vector_mask_operand" " vm,  vm,Wc1, Wc1")
4685              (match_operand 5 "vector_length_operand"    " rK,  rK, rK,  rK")
4686              (match_operand 6 "const_int_operand"        "  i,   i,  i,   i")
4687              (match_operand 7 "const_int_operand"        "  i,   i,  i,   i")
4688              (match_operand 8 "const_int_operand"        "  i,   i,  i,   i")
4689              (reg:SI VL_REGNUM)
4690              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4691           (minus:VI
4692             (match_operand:VI 4 "register_operand"       "  0,  vr,  0,  vr")
4693             (mult:VI
4694               (match_operand:VI 2 "register_operand"     " vr,  vr, vr,  vr")
4695               (match_operand:VI 3 "register_operand"     " vr,  vr, vr,  vr")))
4696           (match_dup 4)))]
4697   "TARGET_VECTOR"
4698   "@
4699    vnmsac.vv\t%0,%2,%3%p1
4700    vmv.v.v\t%0,%4\;vnmsac.vv\t%0,%2,%3%p1
4701    vnmsac.vv\t%0,%2,%3%p1
4702    vmv.v.v\t%0,%4\;vnmsac.vv\t%0,%2,%3%p1"
4703   [(set_attr "type" "vimuladd")
4704    (set_attr "mode" "<MODE>")
4705    (set_attr "merge_op_idx" "2")
4706    (set_attr "vl_op_idx" "5")
4707    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4708    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4709    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4710
4711 (define_insn_and_rewrite "*pred_minus_mul<mode>"
4712   [(set (match_operand:VI 0 "register_operand"            "=&vr,?&vr, ?&vr, ?&vr,  ?&vr")
4713         (if_then_else:VI
4714           (unspec:<VM>
4715             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1,vmWc1")
4716              (match_operand 6 "vector_length_operand"    "   rK,   rK,   rK,   rK,   rK")
4717              (match_operand 7 "const_int_operand"        "    i,    i,    i,    i,    i")
4718              (match_operand 8 "const_int_operand"        "    i,    i,    i,    i,    i")
4719              (match_operand 9 "const_int_operand"        "    i,    i,    i,    i,    i")
4720              (reg:SI VL_REGNUM)
4721              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4722           (minus:VI
4723             (match_operand:VI 4 "vector_arith_operand"   "   vr,   vi,   vr,   vr,   vr")
4724             (mult:VI
4725               (match_operand:VI 2 "register_operand"     "   vr,   vr,   vi,   vr,   vr")
4726               (match_operand:VI 3 "register_operand"     "   vr,   vr,   vr,   vi,   vr")))
4727           (match_operand:VI 5 "register_operand"         "    0,   vr,   vr,   vr,   vr")))]
4728   "TARGET_VECTOR
4729    && !rtx_equal_p (operands[2], operands[5])
4730    && !rtx_equal_p (operands[3], operands[5])
4731    && !rtx_equal_p (operands[4], operands[5])"
4732   "@
4733    vmv.v.v\t%0,%4\;vnmsac.vv\t%0,%2,%3%p1
4734    #
4735    #
4736    #
4737    #"
4738   "&& reload_completed
4739    && !rtx_equal_p (operands[0], operands[5])"
4740   {
4741     if (satisfies_constraint_vi (operands[3]))
4742       std::swap (operands[2], operands[3]);
4743
4744     if (satisfies_constraint_vi (operands[2]))
4745       {
4746         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4747                         operands[5], operands[2], operands[1], operands[6],
4748                         operands[7], operands[9]));
4749         operands[5] = operands[2] = operands[0];
4750       }
4751     else
4752       {
4753         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4754                         operands[5], operands[4], operands[1], operands[6],
4755                         operands[7], operands[9]));
4756         operands[5] = operands[4] = operands[0];
4757       }
4758   }
4759   [(set_attr "type" "vimuladd")
4760    (set_attr "mode" "<MODE>")])
4761
4762 (define_expand "@pred_minus_mul<mode>_scalar"
4763   [(set (match_operand:VI_QHS 0 "register_operand")
4764         (if_then_else:VI_QHS
4765           (unspec:<VM>
4766             [(match_operand:<VM> 1 "vector_mask_operand")
4767              (match_operand 6 "vector_length_operand")
4768              (match_operand 7 "const_int_operand")
4769              (match_operand 8 "const_int_operand")
4770              (match_operand 9 "const_int_operand")
4771              (reg:SI VL_REGNUM)
4772              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4773           (minus:VI_QHS
4774             (match_operand:VI_QHS 4 "register_operand")
4775             (mult:VI_QHS
4776               (vec_duplicate:VI_QHS
4777                 (match_operand:<VEL> 2 "reg_or_int_operand"))
4778               (match_operand:VI_QHS 3 "register_operand")))
4779           (match_operand:VI_QHS 5 "register_operand")))]
4780   "TARGET_VECTOR"
4781 {
4782   operands[2] = force_reg (<VEL>mode, operands[2]);
4783 })
4784
4785 (define_insn "*pred_nmsub<mode>_scalar"
4786   [(set (match_operand:VI 0 "register_operand"            "=vd,?&vd, vr,?&vr")
4787         (if_then_else:VI
4788           (unspec:<VM>
4789             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,  vm,Wc1, Wc1")
4790              (match_operand 5 "vector_length_operand"     " rK,  rK, rK,  rK")
4791              (match_operand 6 "const_int_operand"         "  i,   i,  i,   i")
4792              (match_operand 7 "const_int_operand"         "  i,   i,  i,   i")
4793              (match_operand 8 "const_int_operand"         "  i,   i,  i,   i")
4794              (reg:SI VL_REGNUM)
4795              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4796           (minus:VI
4797             (match_operand:VI 4 "register_operand"        " vr,  vr, vr,  vr")
4798             (mult:VI
4799               (vec_duplicate:VI
4800                 (match_operand:<VEL> 2 "register_operand" "  r,   r,  r,   r"))
4801               (match_operand:VI 3 "register_operand"      "  0,  vr,  0,  vr")))
4802           (match_dup 3)))]
4803   "TARGET_VECTOR"
4804   "@
4805    vnmsub.vx\t%0,%2,%4%p1
4806    vmv.v.v\t%0,%3\;vnmsub.vx\t%0,%2,%4%p1
4807    vnmsub.vx\t%0,%2,%4%p1
4808    vmv.v.v\t%0,%3\;vnmsub.vx\t%0,%2,%4%p1"
4809   [(set_attr "type" "vimuladd")
4810    (set_attr "mode" "<MODE>")
4811    (set_attr "merge_op_idx" "4")
4812    (set_attr "vl_op_idx" "5")
4813    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4814    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4815    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4816
4817 (define_insn "*pred_nmsac<mode>_scalar"
4818   [(set (match_operand:VI 0 "register_operand"            "=vd,?&vd, vr,?&vr")
4819         (if_then_else:VI
4820           (unspec:<VM>
4821             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,  vm,Wc1, Wc1")
4822              (match_operand 5 "vector_length_operand"     " rK,  rK, rK,  rK")
4823              (match_operand 6 "const_int_operand"         "  i,   i,  i,   i")
4824              (match_operand 7 "const_int_operand"         "  i,   i,  i,   i")
4825              (match_operand 8 "const_int_operand"         "  i,   i,  i,   i")
4826              (reg:SI VL_REGNUM)
4827              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4828           (minus:VI
4829             (match_operand:VI 4 "register_operand"        "  0,  vr,  0,  vr")
4830             (mult:VI
4831               (vec_duplicate:VI
4832                 (match_operand:<VEL> 2 "register_operand" "  r,   r,  r,   r"))
4833               (match_operand:VI 3 "register_operand"      " vr,  vr, vr,  vr")))
4834           (match_dup 4)))]
4835   "TARGET_VECTOR"
4836   "@
4837    vnmsac.vx\t%0,%2,%3%p1
4838    vmv.v.v\t%0,%4\;vnmsac.vx\t%0,%2,%3%p1
4839    vnmsac.vx\t%0,%2,%3%p1
4840    vmv.v.v\t%0,%4\;vnmsac.vx\t%0,%2,%3%p1"
4841   [(set_attr "type" "vimuladd")
4842    (set_attr "mode" "<MODE>")
4843    (set_attr "merge_op_idx" "2")
4844    (set_attr "vl_op_idx" "5")
4845    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4846    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4847    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4848
4849 (define_insn_and_rewrite "*pred_minus_mul<mode>_scalar"
4850   [(set (match_operand:VI 0 "register_operand"            "=&vr, ?&vr, ?&vr, ?&vr")
4851         (if_then_else:VI
4852           (unspec:<VM>
4853             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1")
4854              (match_operand 6 "vector_length_operand"    "   rK,   rK,   rK,   rK")
4855              (match_operand 7 "const_int_operand"        "    i,    i,    i,    i")
4856              (match_operand 8 "const_int_operand"        "    i,    i,    i,    i")
4857              (match_operand 9 "const_int_operand"        "    i,    i,    i,    i")
4858              (reg:SI VL_REGNUM)
4859              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4860           (minus:VI
4861             (match_operand:VI 4 "vector_arith_operand"    "   vr,   vi,   vr,   vr")
4862             (mult:VI
4863               (vec_duplicate:VI
4864                 (match_operand:<VEL> 2 "register_operand" "    r,    r,    r,    r"))
4865               (match_operand:VI 3 "register_operand"      "   vr,   vr,   vi,   vr")))
4866           (match_operand:VI 5 "register_operand"          "    0,   vr,   vr,   vr")))]
4867   "TARGET_VECTOR
4868    && !rtx_equal_p (operands[3], operands[5])
4869    && !rtx_equal_p (operands[4], operands[5])"
4870   "@
4871    vmv.v.v\t%0,%4\;vnmsac.vx\t%0,%2,%3%p1
4872    #
4873    #
4874    #"
4875   "&& reload_completed
4876    && !rtx_equal_p (operands[0], operands[5])"
4877   {
4878     if (satisfies_constraint_vi (operands[3]))
4879       {
4880         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4881                         operands[5], operands[3], operands[1], operands[6],
4882                         operands[7], operands[9]));
4883         operands[5] = operands[3] = operands[0];
4884       }
4885     else
4886       {
4887         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
4888                         operands[5], operands[4], operands[1], operands[6],
4889                         operands[7], operands[9]));
4890         operands[5] = operands[4] = operands[0];
4891       }
4892   }
4893   [(set_attr "type" "vimuladd")
4894    (set_attr "mode" "<MODE>")])
4895
4896 (define_expand "@pred_minus_mul<mode>_scalar"
4897   [(set (match_operand:VI_D 0 "register_operand")
4898         (if_then_else:VI_D
4899           (unspec:<VM>
4900             [(match_operand:<VM> 1 "vector_mask_operand")
4901              (match_operand 6 "vector_length_operand")
4902              (match_operand 7 "const_int_operand")
4903              (match_operand 8 "const_int_operand")
4904              (match_operand 9 "const_int_operand")
4905              (reg:SI VL_REGNUM)
4906              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4907           (minus:VI_D
4908             (match_operand:VI_D 4 "register_operand")
4909             (mult:VI_D
4910               (vec_duplicate:VI_D
4911                 (match_operand:<VEL> 2 "reg_or_int_operand"))
4912               (match_operand:VI_D 3 "register_operand")))
4913           (match_operand:VI_D 5 "register_operand")))]
4914   "TARGET_VECTOR"
4915 {
4916   if (riscv_vector::sew64_scalar_helper (
4917         operands,
4918         /* scalar op */&operands[2],
4919         /* vl */operands[6],
4920         <MODE>mode,
4921         <VM>mode,
4922         false,
4923         [] (rtx *operands, rtx boardcast_scalar) {
4924           emit_insn (gen_pred_minus_mul<mode> (operands[0], operands[1],
4925                boardcast_scalar, operands[3], operands[4], operands[5],
4926                operands[6], operands[7], operands[8], operands[9]));
4927         }))
4928     DONE;
4929 })
4930
4931 (define_insn "*pred_nmsub<mode>_extended_scalar"
4932   [(set (match_operand:VI_D 0 "register_operand"               "=vd,?&vd, vr,?&vr")
4933         (if_then_else:VI_D
4934           (unspec:<VM>
4935             [(match_operand:<VM> 1 "vector_mask_operand"       " vm,  vm,Wc1, Wc1")
4936              (match_operand 5 "vector_length_operand"          " rK,  rK, rK,  rK")
4937              (match_operand 6 "const_int_operand"              "  i,   i,  i,   i")
4938              (match_operand 7 "const_int_operand"              "  i,   i,  i,   i")
4939              (match_operand 8 "const_int_operand"              "  i,   i,  i,   i")
4940              (reg:SI VL_REGNUM)
4941              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4942           (minus:VI_D
4943             (match_operand:VI_D 4 "register_operand"           " vr,  vr, vr,  vr")
4944             (mult:VI_D
4945               (vec_duplicate:VI_D
4946                 (sign_extend:<VEL>
4947                   (match_operand:<VSUBEL> 2 "register_operand" "  r,   r,  r,   r")))
4948               (match_operand:VI_D 3 "register_operand"         "  0,  vr,  0,  vr")))
4949           (match_dup 3)))]
4950   "TARGET_VECTOR"
4951   "@
4952    vnmsub.vx\t%0,%2,%4%p1
4953    vmv.v.v\t%0,%3\;vnmsub.vx\t%0,%2,%4%p1
4954    vnmsub.vx\t%0,%2,%4%p1
4955    vmv.v.v\t%0,%3\;vnmsub.vx\t%0,%2,%4%p1"
4956   [(set_attr "type" "vimuladd")
4957    (set_attr "mode" "<MODE>")
4958    (set_attr "merge_op_idx" "4")
4959    (set_attr "vl_op_idx" "5")
4960    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4961    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4962    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4963
4964 (define_insn "*pred_nmsac<mode>_extended_scalar"
4965   [(set (match_operand:VI_D 0 "register_operand"               "=vd,?&vd, vr,?&vr")
4966         (if_then_else:VI_D
4967           (unspec:<VM>
4968             [(match_operand:<VM> 1 "vector_mask_operand"       " vm,  vm,Wc1, Wc1")
4969              (match_operand 5 "vector_length_operand"          " rK,  rK, rK,  rK")
4970              (match_operand 6 "const_int_operand"              "  i,   i,  i,   i")
4971              (match_operand 7 "const_int_operand"              "  i,   i,  i,   i")
4972              (match_operand 8 "const_int_operand"              "  i,   i,  i,   i")
4973              (reg:SI VL_REGNUM)
4974              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
4975           (minus:VI_D
4976             (match_operand:VI_D 4 "register_operand"           "  0,  vr,  0,  vr")
4977             (mult:VI_D
4978               (vec_duplicate:VI_D
4979                 (sign_extend:<VEL>
4980                   (match_operand:<VSUBEL> 2 "register_operand" "  r,   r,  r,   r")))
4981               (match_operand:VI_D 3 "register_operand"         " vr,  vr, vr,  vr")))
4982           (match_dup 4)))]
4983   "TARGET_VECTOR"
4984   "@
4985    vnmsac.vx\t%0,%2,%3%p1
4986    vmv.v.v\t%0,%4\;vnmsac.vx\t%0,%2,%3%p1
4987    vnmsac.vx\t%0,%2,%3%p1
4988    vmv.v.v\t%0,%4\;vnmsac.vx\t%0,%2,%3%p1"
4989   [(set_attr "type" "vimuladd")
4990    (set_attr "mode" "<MODE>")
4991    (set_attr "merge_op_idx" "2")
4992    (set_attr "vl_op_idx" "5")
4993    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
4994    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
4995    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
4996
4997 (define_insn_and_rewrite "*pred_minus_mul<mode>_extended_scalar"
4998   [(set (match_operand:VI_D 0 "register_operand"                "=&vr, ?&vr, ?&vr, ?&vr")
4999         (if_then_else:VI_D
5000           (unspec:<VM>
5001             [(match_operand:<VM> 1 "vector_mask_operand"       "vmWc1,vmWc1,vmWc1,vmWc1")
5002              (match_operand 6 "vector_length_operand"          "   rK,   rK,   rK,   rK")
5003              (match_operand 7 "const_int_operand"              "    i,    i,    i,    i")
5004              (match_operand 8 "const_int_operand"              "    i,    i,    i,    i")
5005              (match_operand 9 "const_int_operand"              "    i,    i,    i,    i")
5006              (reg:SI VL_REGNUM)
5007              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5008           (minus:VI_D
5009             (match_operand:VI_D 4 "vector_arith_operand"       "   vr,   vr,   vr,   vr")
5010             (mult:VI_D
5011               (vec_duplicate:VI_D
5012                 (sign_extend:<VEL>
5013                   (match_operand:<VSUBEL> 2 "register_operand" "    r,    r,    r,    r")))
5014               (match_operand:VI_D 3 "register_operand"         "   vr,   vr,   vr,   vr")))
5015           (match_operand:VI_D 5 "register_operand"             "    0,   vr,   vr,   vr")))]
5016   "TARGET_VECTOR
5017    && !rtx_equal_p (operands[3], operands[5])
5018    && !rtx_equal_p (operands[4], operands[5])"
5019   "@
5020    vmv.v.v\t%0,%4\;vnmsac.vx\t%0,%2,%3%p1
5021    #
5022    #
5023    #"
5024   "&& reload_completed
5025    && !rtx_equal_p (operands[0], operands[5])"
5026   {
5027     if (satisfies_constraint_vi (operands[3]))
5028       {
5029         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
5030                         operands[5], operands[3], operands[1], operands[6],
5031                         operands[7], operands[9]));
5032         operands[5] = operands[3] = operands[0];
5033       }
5034     else
5035       {
5036         emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
5037                         operands[5], operands[4], operands[1], operands[6],
5038                         operands[7], operands[9]));
5039         operands[5] = operands[4] = operands[0];
5040       }
5041   }
5042   [(set_attr "type" "vimuladd")
5043    (set_attr "mode" "<MODE>")])
5044
5045 ;; -------------------------------------------------------------------------------
5046 ;; ---- Predicated widen integer ternary operations
5047 ;; -------------------------------------------------------------------------------
5048 ;; Includes:
5049 ;; - 11.14 Vector Widening Integer Multiply-Add Instructions
5050 ;; -------------------------------------------------------------------------------
5051
5052 (define_insn "@pred_widen_mul_plus<su><mode>"
5053   [(set (match_operand:VWEXTI 0 "register_operand"                    "=&vr")
5054         (if_then_else:VWEXTI
5055           (unspec:<VM>
5056             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1")
5057              (match_operand 5 "vector_length_operand"                "   rK")
5058              (match_operand 6 "const_int_operand"                    "    i")
5059              (match_operand 7 "const_int_operand"                    "    i")
5060              (match_operand 8 "const_int_operand"                    "    i")
5061              (reg:SI VL_REGNUM)
5062              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5063           (plus:VWEXTI
5064             (mult:VWEXTI
5065               (any_extend:VWEXTI
5066                 (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr"))
5067               (any_extend:VWEXTI
5068                 (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
5069             (match_operand:VWEXTI 2 "register_operand"               "    0"))
5070           (match_dup 2)))]
5071   "TARGET_VECTOR"
5072   "vwmacc<u>.vv\t%0,%3,%4%p1"
5073   [(set_attr "type" "viwmuladd")
5074    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
5075
5076 (define_insn "@pred_widen_mul_plus<su><mode>_scalar"
5077   [(set (match_operand:VWEXTI 0 "register_operand"                    "=&vr")
5078         (if_then_else:VWEXTI
5079           (unspec:<VM>
5080             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1")
5081              (match_operand 5 "vector_length_operand"                "   rK")
5082              (match_operand 6 "const_int_operand"                    "    i")
5083              (match_operand 7 "const_int_operand"                    "    i")
5084              (match_operand 8 "const_int_operand"                    "    i")
5085              (reg:SI VL_REGNUM)
5086              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5087           (plus:VWEXTI
5088             (mult:VWEXTI
5089               (any_extend:VWEXTI
5090                 (vec_duplicate:<V_DOUBLE_TRUNC>
5091                   (match_operand:<VSUBEL> 3 "register_operand"       "    r")))
5092               (any_extend:VWEXTI
5093                 (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
5094             (match_operand:VWEXTI 2 "register_operand"               "    0"))
5095           (match_dup 2)))]
5096   "TARGET_VECTOR"
5097   "vwmacc<u>.vx\t%0,%3,%4%p1"
5098   [(set_attr "type" "viwmuladd")
5099    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
5100
5101 (define_insn "@pred_widen_mul_plussu<mode>"
5102   [(set (match_operand:VWEXTI 0 "register_operand"                    "=&vr")
5103         (if_then_else:VWEXTI
5104           (unspec:<VM>
5105             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1")
5106              (match_operand 5 "vector_length_operand"                "   rK")
5107              (match_operand 6 "const_int_operand"                    "    i")
5108              (match_operand 7 "const_int_operand"                    "    i")
5109              (match_operand 8 "const_int_operand"                    "    i")
5110              (reg:SI VL_REGNUM)
5111              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5112           (plus:VWEXTI
5113             (mult:VWEXTI
5114               (sign_extend:VWEXTI
5115                 (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr"))
5116               (zero_extend:VWEXTI
5117                 (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
5118             (match_operand:VWEXTI 2 "register_operand"               "    0"))
5119           (match_dup 2)))]
5120   "TARGET_VECTOR"
5121   "vwmaccsu.vv\t%0,%3,%4%p1"
5122   [(set_attr "type" "viwmuladd")
5123    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
5124
5125 (define_insn "@pred_widen_mul_plussu<mode>_scalar"
5126   [(set (match_operand:VWEXTI 0 "register_operand"                    "=&vr")
5127         (if_then_else:VWEXTI
5128           (unspec:<VM>
5129             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1")
5130              (match_operand 5 "vector_length_operand"                "   rK")
5131              (match_operand 6 "const_int_operand"                    "    i")
5132              (match_operand 7 "const_int_operand"                    "    i")
5133              (match_operand 8 "const_int_operand"                    "    i")
5134              (reg:SI VL_REGNUM)
5135              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5136           (plus:VWEXTI
5137             (mult:VWEXTI
5138               (sign_extend:VWEXTI
5139                 (vec_duplicate:<V_DOUBLE_TRUNC>
5140                   (match_operand:<VSUBEL> 3 "register_operand"       "    r")))
5141               (zero_extend:VWEXTI
5142                 (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
5143             (match_operand:VWEXTI 2 "register_operand"               "    0"))
5144           (match_dup 2)))]
5145   "TARGET_VECTOR"
5146   "vwmaccsu.vx\t%0,%3,%4%p1"
5147   [(set_attr "type" "viwmuladd")
5148    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
5149
5150 (define_insn "@pred_widen_mul_plusus<mode>_scalar"
5151   [(set (match_operand:VWEXTI 0 "register_operand"                    "=&vr")
5152         (if_then_else:VWEXTI
5153           (unspec:<VM>
5154             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1")
5155              (match_operand 5 "vector_length_operand"                "   rK")
5156              (match_operand 6 "const_int_operand"                    "    i")
5157              (match_operand 7 "const_int_operand"                    "    i")
5158              (match_operand 8 "const_int_operand"                    "    i")
5159              (reg:SI VL_REGNUM)
5160              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5161           (plus:VWEXTI
5162             (mult:VWEXTI
5163               (zero_extend:VWEXTI
5164                 (vec_duplicate:<V_DOUBLE_TRUNC>
5165                   (match_operand:<VSUBEL> 3 "register_operand"       "    r")))
5166               (sign_extend:VWEXTI
5167                 (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
5168             (match_operand:VWEXTI 2 "register_operand"               "    0"))
5169           (match_dup 2)))]
5170   "TARGET_VECTOR"
5171   "vwmaccus.vx\t%0,%3,%4%p1"
5172   [(set_attr "type" "viwmuladd")
5173    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
5174
5175 ;; -------------------------------------------------------------------------------
5176 ;; ---- Predicated BOOL mask operations
5177 ;; -------------------------------------------------------------------------------
5178 ;; Includes:
5179 ;; - 15.1 Vector Mask-Register Logical Instructions
5180 ;; - 15.2 Vector count population in mask vcpop.m
5181 ;; - 15.3 vfirst find-first-set mask bit
5182 ;; - 15.4 vmsbf.m set-before-first mask bit
5183 ;; - 15.5 vmsif.m set-including-first mask bit
5184 ;; - 15.6 vmsof.m set-only-first mask bit
5185 ;; - 15.8 Vector Iota Instruction
5186 ;; - 15.9 Vector Element Index Instruction
5187 ;; -------------------------------------------------------------------------------
5188
5189 ;; We keep this pattern same as pred_mov so that we can gain more optimizations.
5190 ;; For example, if we have vmxor.mm v1,v1,v1. It will be optmized as vmclr.m which
5191 ;; is generated by pred_mov.
5192 (define_insn "@pred_<optab><mode>"
5193   [(set (match_operand:VB 0 "register_operand"                   "=vr")
5194         (if_then_else:VB
5195           (unspec:VB
5196             [(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1")
5197              (match_operand 5 "vector_length_operand"            " rK")
5198              (match_operand 6 "const_int_operand"                "  i")
5199              (reg:SI VL_REGNUM)
5200              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5201           (any_bitwise:VB
5202             (match_operand:VB 3 "register_operand"               " vr")
5203             (match_operand:VB 4 "register_operand"               " vr"))
5204           (match_operand:VB 2 "vector_undef_operand"             " vu")))]
5205   "TARGET_VECTOR"
5206   "vm<insn>.mm\t%0,%3,%4"
5207   [(set_attr "type" "vmalu")
5208    (set_attr "mode" "<MODE>")
5209    (set_attr "vl_op_idx" "5")
5210    (set (attr "avl_type") (symbol_ref "INTVAL (operands[6])"))])
5211
5212 (define_insn "@pred_n<optab><mode>"
5213   [(set (match_operand:VB 0 "register_operand"                   "=vr")
5214         (if_then_else:VB
5215           (unspec:VB
5216             [(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1")
5217              (match_operand 5 "vector_length_operand"            " rK")
5218              (match_operand 6 "const_int_operand"                "  i")
5219              (reg:SI VL_REGNUM)
5220              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5221           (not:VB
5222             (any_bitwise:VB
5223               (match_operand:VB 3 "register_operand"             " vr")
5224               (match_operand:VB 4 "register_operand"             " vr")))
5225           (match_operand:VB 2 "vector_undef_operand"             " vu")))]
5226   "TARGET_VECTOR"
5227   "vm<ninsn>.mm\t%0,%3,%4"
5228   [(set_attr "type" "vmalu")
5229    (set_attr "mode" "<MODE>")
5230    (set_attr "vl_op_idx" "5")
5231    (set (attr "avl_type") (symbol_ref "INTVAL (operands[6])"))])
5232
5233 (define_insn "@pred_<optab>not<mode>"
5234   [(set (match_operand:VB 0 "register_operand"                   "=vr")
5235         (if_then_else:VB
5236           (unspec:VB
5237             [(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1")
5238              (match_operand 5 "vector_length_operand"            " rK")
5239              (match_operand 6 "const_int_operand"                "  i")
5240              (reg:SI VL_REGNUM)
5241              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5242           (and_ior:VB
5243             (match_operand:VB 3 "register_operand"               " vr")
5244             (not:VB
5245               (match_operand:VB 4 "register_operand"             " vr")))
5246           (match_operand:VB 2 "vector_undef_operand"             " vu")))]
5247   "TARGET_VECTOR"
5248   "vm<insn>n.mm\t%0,%3,%4"
5249   [(set_attr "type" "vmalu")
5250    (set_attr "mode" "<MODE>")
5251    (set_attr "vl_op_idx" "5")
5252    (set (attr "avl_type") (symbol_ref "INTVAL (operands[6])"))])
5253
5254 (define_insn "@pred_not<mode>"
5255   [(set (match_operand:VB 0 "register_operand"                   "=vr")
5256         (if_then_else:VB
5257           (unspec:VB
5258             [(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1")
5259              (match_operand 4 "vector_length_operand"            " rK")
5260              (match_operand 5 "const_int_operand"                "  i")
5261              (reg:SI VL_REGNUM)
5262              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5263           (not:VB
5264             (match_operand:VB 3 "register_operand"               " vr"))
5265           (match_operand:VB 2 "vector_undef_operand"             " vu")))]
5266   "TARGET_VECTOR"
5267   "vmnot.m\t%0,%3"
5268   [(set_attr "type" "vmalu")
5269    (set_attr "mode" "<MODE>")
5270    (set_attr "vl_op_idx" "4")
5271    (set (attr "avl_type") (symbol_ref "INTVAL (operands[5])"))])
5272
5273 (define_insn "@pred_popcount<VB:mode><P:mode>"
5274   [(set (match_operand:P 0 "register_operand"               "=r")
5275         (popcount:P
5276           (unspec:VB
5277             [(and:VB
5278                (match_operand:VB 1 "vector_mask_operand" "vmWc1")
5279                (match_operand:VB 2 "register_operand"    "   vr"))
5280              (match_operand 3 "vector_length_operand"    "   rK")
5281              (match_operand 4 "const_int_operand"        "    i")
5282              (reg:SI VL_REGNUM)
5283              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)))]
5284   "TARGET_VECTOR"
5285   "vcpop.m\t%0,%2%p1"
5286   [(set_attr "type" "vmpop")
5287    (set_attr "mode" "<VB:MODE>")])
5288
5289 (define_insn "@pred_ffs<VB:mode><P:mode>"
5290   [(set (match_operand:P 0 "register_operand"                 "=r")
5291         (plus:P
5292           (ffs:P
5293             (unspec:VB
5294               [(and:VB
5295                  (match_operand:VB 1 "vector_mask_operand" "vmWc1")
5296                  (match_operand:VB 2 "register_operand"    "   vr"))
5297                (match_operand 3 "vector_length_operand"    "   rK")
5298                (match_operand 4 "const_int_operand"        "    i")
5299                (reg:SI VL_REGNUM)
5300                (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE))
5301           (const_int -1)))]
5302   "TARGET_VECTOR"
5303   "vfirst.m\t%0,%2%p1"
5304   [(set_attr "type" "vmffs")
5305    (set_attr "mode" "<VB:MODE>")])
5306
5307 (define_insn "@pred_<misc_op><mode>"
5308   [(set (match_operand:VB 0 "register_operand"          "=&vr,  &vr")
5309         (if_then_else:VB
5310           (unspec:VB
5311             [(match_operand:VB 1 "vector_mask_operand" "vmWc1,vmWc1")
5312              (match_operand 4 "vector_length_operand"  "   rK,   rK")
5313              (match_operand 5 "const_int_operand"      "    i,    i")
5314              (match_operand 6 "const_int_operand"      "    i,    i")
5315              (reg:SI VL_REGNUM)
5316              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5317           (unspec:VB
5318             [(match_operand:VB 3 "register_operand"    "   vr,   vr")] VMISC)
5319           (match_operand:VB 2 "vector_merge_operand"   "   vu,    0")))]
5320   "TARGET_VECTOR"
5321   "vm<misc_op>.m\t%0,%3%p1"
5322   [(set_attr "type" "vmsfs")
5323    (set_attr "mode" "<MODE>")])
5324
5325 (define_insn "@pred_iota<mode>"
5326   [(set (match_operand:VI 0 "register_operand"            "=&vr,  &vr")
5327         (if_then_else:VI
5328           (unspec:<VM>
5329             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1")
5330              (match_operand 4 "vector_length_operand"    "   rK,   rK")
5331              (match_operand 5 "const_int_operand"        "    i,    i")
5332              (match_operand 6 "const_int_operand"        "    i,    i")
5333              (match_operand 7 "const_int_operand"        "    i,    i")
5334              (reg:SI VL_REGNUM)
5335              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5336           (unspec:VI
5337             [(match_operand:<VM> 3 "register_operand"    "   vr,   vr")] UNSPEC_VIOTA)
5338           (match_operand:VI 2 "vector_merge_operand"     "   vu,    0")))]
5339   "TARGET_VECTOR"
5340   "viota.m\t%0,%3%p1"
5341   [(set_attr "type" "vmiota")
5342    (set_attr "mode" "<MODE>")])
5343
5344 (define_insn "@pred_series<mode>"
5345   [(set (match_operand:VI 0 "register_operand"           "=vd, vd, vr, vr")
5346         (if_then_else:VI
5347           (unspec:<VM>
5348             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
5349              (match_operand 3 "vector_length_operand"    " rK, rK, rK, rK")
5350              (match_operand 4 "const_int_operand"        "  i,  i,  i,  i")
5351              (match_operand 5 "const_int_operand"        "  i,  i,  i,  i")
5352              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
5353              (reg:SI VL_REGNUM)
5354              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5355           (vec_series:VI (const_int 0) (const_int 1))
5356           (match_operand:VI 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
5357   "TARGET_VECTOR"
5358   "vid.v\t%0%p1"
5359   [(set_attr "type" "vmidx")
5360    (set_attr "mode" "<MODE>")])
5361
5362 ;; -------------------------------------------------------------------------------
5363 ;; ---- Predicated floating-point binary operations
5364 ;; -------------------------------------------------------------------------------
5365 ;; Includes:
5366 ;; - 13.2 Vector Single-Width Floating-Point Add/Subtract Instructions
5367 ;; - 13.4 Vector Single-Width Floating-Point Multiply/Divide Instructions
5368 ;; - 13.11 Vector Floating-Point MIN/MAX Instructions
5369 ;; - 13.12 Vector Floating-Point Sign-Injection Instructions
5370 ;; -------------------------------------------------------------------------------
5371
5372 (define_insn "@pred_<optab><mode>"
5373   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
5374         (if_then_else:VF
5375           (unspec:<VM>
5376             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
5377              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
5378              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
5379              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
5380              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
5381              (reg:SI VL_REGNUM)
5382              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5383           (any_float_binop:VF
5384             (match_operand:VF 3 "register_operand"       " vr, vr, vr, vr")
5385             (match_operand:VF 4 "register_operand"       " vr, vr, vr, vr"))
5386           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
5387   "TARGET_VECTOR"
5388   "vf<insn>.vv\t%0,%3,%4%p1"
5389   [(set_attr "type" "<float_insn_type>")
5390    (set_attr "mode" "<MODE>")])
5391
5392 (define_insn "@pred_<optab><mode>_scalar"
5393   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
5394         (if_then_else:VF
5395           (unspec:<VM>
5396             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
5397              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
5398              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
5399              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
5400              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
5401              (reg:SI VL_REGNUM)
5402              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5403           (commutative_float_binop:VF
5404             (vec_duplicate:VF
5405               (match_operand:<VEL> 4 "register_operand"  "  f,  f,  f,  f"))
5406             (match_operand:VF 3 "register_operand"       " vr, vr, vr, vr"))
5407           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
5408   "TARGET_VECTOR"
5409   "vf<insn>.vf\t%0,%3,%4%p1"
5410   [(set_attr "type" "<float_insn_type>")
5411    (set_attr "mode" "<MODE>")])
5412
5413 (define_insn "@pred_<optab><mode>_scalar"
5414   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
5415         (if_then_else:VF
5416           (unspec:<VM>
5417             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
5418              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
5419              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
5420              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
5421              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
5422              (reg:SI VL_REGNUM)
5423              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5424           (non_commutative_float_binop:VF
5425             (match_operand:VF 3 "register_operand"       " vr, vr, vr, vr")
5426             (vec_duplicate:VF
5427               (match_operand:<VEL> 4 "register_operand"  "  f,  f,  f,  f")))
5428           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
5429   "TARGET_VECTOR"
5430   "vf<insn>.vf\t%0,%3,%4%p1"
5431   [(set_attr "type" "<float_insn_type>")
5432    (set_attr "mode" "<MODE>")])
5433
5434 (define_insn "@pred_<optab><mode>_reverse_scalar"
5435   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
5436         (if_then_else:VF
5437           (unspec:<VM>
5438             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
5439              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
5440              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
5441              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
5442              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
5443              (reg:SI VL_REGNUM)
5444              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5445           (non_commutative_float_binop:VF
5446             (vec_duplicate:VF
5447               (match_operand:<VEL> 4 "register_operand"  "  f,  f,  f,  f"))
5448             (match_operand:VF 3 "register_operand"       " vr, vr, vr, vr"))
5449           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
5450   "TARGET_VECTOR"
5451   "vfr<insn>.vf\t%0,%3,%4%p1"
5452   [(set_attr "type" "<float_insn_type>")
5453    (set_attr "mode" "<MODE>")])
5454
5455 (define_insn "@pred_<copysign><mode>"
5456   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
5457         (if_then_else:VF
5458           (unspec:<VM>
5459             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
5460              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
5461              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
5462              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
5463              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
5464              (reg:SI VL_REGNUM)
5465              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5466           (unspec:VF
5467             [(match_operand:VF 3 "register_operand"       " vr, vr, vr, vr")
5468              (match_operand:VF 4 "register_operand"       " vr, vr, vr, vr")] VCOPYSIGNS)
5469           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
5470   "TARGET_VECTOR"
5471   "vfsgnj<nx>.vv\t%0,%3,%4%p1"
5472   [(set_attr "type" "vfsgnj")
5473    (set_attr "mode" "<MODE>")])
5474
5475 (define_insn "@pred_<copysign><mode>_scalar"
5476   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
5477         (if_then_else:VF
5478           (unspec:<VM>
5479             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
5480              (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
5481              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
5482              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
5483              (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
5484              (reg:SI VL_REGNUM)
5485              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5486           (unspec:VF
5487             [(match_operand:VF 3 "register_operand"      " vr, vr, vr, vr")
5488              (vec_duplicate:VF
5489                (match_operand:<VEL> 4 "register_operand" "  f,  f,  f,  f"))] VCOPYSIGNS)
5490           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
5491   "TARGET_VECTOR"
5492   "vfsgnj<nx>.vf\t%0,%3,%4%p1"
5493   [(set_attr "type" "vfsgnj")
5494    (set_attr "mode" "<MODE>")])
5495
5496 ;; -------------------------------------------------------------------------------
5497 ;; ---- Predicated floating-point ternary operations
5498 ;; -------------------------------------------------------------------------------
5499 ;; Includes:
5500 ;; - 13.6 Vector Single-Width Floating-Point Fused Multiply-Add Instructions
5501 ;; -------------------------------------------------------------------------------
5502
5503 (define_expand "@pred_mul_<optab><mode>"
5504   [(set (match_operand:VF 0 "register_operand")
5505         (if_then_else:VF
5506           (unspec:<VM>
5507             [(match_operand:<VM> 1 "vector_mask_operand")
5508              (match_operand 6 "vector_length_operand")
5509              (match_operand 7 "const_int_operand")
5510              (match_operand 8 "const_int_operand")
5511              (match_operand 9 "const_int_operand")
5512              (reg:SI VL_REGNUM)
5513              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5514           (plus_minus:VF
5515             (mult:VF
5516               (match_operand:VF 2 "register_operand")
5517               (match_operand:VF 3 "register_operand"))
5518             (match_operand:VF 4 "register_operand"))
5519           (match_operand:VF 5 "register_operand")))]
5520   "TARGET_VECTOR"
5521 {
5522   /* Swap the multiplication operands if the fallback value is the
5523      second of the two.  */
5524   if (rtx_equal_p (operands[3], operands[5]))
5525     std::swap (operands[2], operands[3]);
5526 })
5527
5528 (define_insn "*pred_<madd_msub><mode>"
5529   [(set (match_operand:VF 0 "register_operand"           "=vd, ?&vd, vr, ?&vr")
5530         (if_then_else:VF
5531           (unspec:<VM>
5532             [(match_operand:<VM> 1 "vector_mask_operand" " vm,   vm,Wc1,  Wc1")
5533              (match_operand 5 "vector_length_operand"    " rK,   rK, rK,   rK")
5534              (match_operand 6 "const_int_operand"        "  i,    i,  i,    i")
5535              (match_operand 7 "const_int_operand"        "  i,    i,  i,    i")
5536              (match_operand 8 "const_int_operand"        "  i,    i,  i,    i")
5537              (reg:SI VL_REGNUM)
5538              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5539           (plus_minus:VF
5540             (mult:VF
5541               (match_operand:VF 2 "register_operand"     "  0,   vr,  0,   vr")
5542               (match_operand:VF 3 "register_operand"     " vr,   vr, vr,   vr"))
5543             (match_operand:VF 4 "register_operand"       " vr,   vr, vr,   vr"))
5544           (match_dup 2)))]
5545   "TARGET_VECTOR"
5546   "@
5547    vf<madd_msub>.vv\t%0,%3,%4%p1
5548    vmv.v.v\t%0,%2\;vf<madd_msub>.vv\t%0,%3,%4%p1
5549    vf<madd_msub>.vv\t%0,%3,%4%p1
5550    vmv.v.v\t%0,%2\;vf<madd_msub>.vv\t%0,%3,%4%p1"
5551   [(set_attr "type" "vfmuladd")
5552    (set_attr "mode" "<MODE>")
5553    (set_attr "merge_op_idx" "4")
5554    (set_attr "vl_op_idx" "5")
5555    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5556    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5557    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5558
5559 (define_insn "*pred_<macc_msac><mode>"
5560   [(set (match_operand:VF 0 "register_operand"           "=vd, ?&vd, vr, ?&vr")
5561         (if_then_else:VF
5562           (unspec:<VM>
5563             [(match_operand:<VM> 1 "vector_mask_operand" " vm,   vm,Wc1,  Wc1")
5564              (match_operand 5 "vector_length_operand"    " rK,   rK, rK,   rK")
5565              (match_operand 6 "const_int_operand"        "  i,    i,  i,    i")
5566              (match_operand 7 "const_int_operand"        "  i,    i,  i,    i")
5567              (match_operand 8 "const_int_operand"        "  i,    i,  i,    i")
5568              (reg:SI VL_REGNUM)
5569              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5570           (plus_minus:VF
5571             (mult:VF
5572               (match_operand:VF 2 "register_operand"     " vr,   vr, vr,   vr")
5573               (match_operand:VF 3 "register_operand"     " vr,   vr, vr,   vr"))
5574             (match_operand:VF 4 "register_operand"       "  0,   vr,  0,   vr"))
5575           (match_dup 4)))]
5576   "TARGET_VECTOR"
5577   "@
5578    vf<macc_msac>.vv\t%0,%2,%3%p1
5579    vmv.v.v\t%0,%4\;vf<macc_msac>.vv\t%0,%2,%3%p1
5580    vf<macc_msac>.vv\t%0,%2,%3%p1
5581    vmv.v.v\t%0,%4\;vf<macc_msac>.vv\t%0,%2,%3%p1"
5582   [(set_attr "type" "vfmuladd")
5583    (set_attr "mode" "<MODE>")
5584    (set_attr "merge_op_idx" "2")
5585    (set_attr "vl_op_idx" "5")
5586    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5587    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5588    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5589
5590 (define_insn_and_rewrite "*pred_mul_<optab><mode>"
5591   [(set (match_operand:VF 0 "register_operand"            "=&vr, ?&vr")
5592         (if_then_else:VF
5593           (unspec:<VM>
5594             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1")
5595              (match_operand 6 "vector_length_operand"    "   rK,   rK")
5596              (match_operand 7 "const_int_operand"        "    i,    i")
5597              (match_operand 8 "const_int_operand"        "    i,    i")
5598              (match_operand 9 "const_int_operand"        "    i,    i")
5599              (reg:SI VL_REGNUM)
5600              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5601           (plus_minus:VF
5602             (mult:VF
5603               (match_operand:VF 2 "register_operand"     "   vr,   vr")
5604               (match_operand:VF 3 "register_operand"     "   vr,   vr"))
5605             (match_operand:VF 4 "vector_arith_operand"   "   vr,   vr"))
5606           (match_operand:VF 5 "register_operand"         "    0,   vr")))]
5607   "TARGET_VECTOR
5608    && !rtx_equal_p (operands[2], operands[5])
5609    && !rtx_equal_p (operands[3], operands[5])
5610    && !rtx_equal_p (operands[4], operands[5])"
5611   "@
5612    vmv.v.v\t%0,%4\;vf<macc_msac>.vv\t%0,%2,%3%p1
5613    #"
5614   "&& reload_completed
5615    && !rtx_equal_p (operands[0], operands[5])"
5616   {
5617     emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
5618                         operands[5], operands[4], operands[1], operands[6],
5619                         operands[7], operands[9]));
5620     operands[5] = operands[4] = operands[0];
5621   }
5622   [(set_attr "type" "vfmuladd")
5623    (set_attr "mode" "<MODE>")])
5624
5625 (define_expand "@pred_mul_<optab><mode>_scalar"
5626   [(set (match_operand:VF 0 "register_operand")
5627         (if_then_else:VF
5628           (unspec:<VM>
5629             [(match_operand:<VM> 1 "vector_mask_operand")
5630              (match_operand 6 "vector_length_operand")
5631              (match_operand 7 "const_int_operand")
5632              (match_operand 8 "const_int_operand")
5633              (match_operand 9 "const_int_operand")
5634              (reg:SI VL_REGNUM)
5635              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5636           (plus_minus:VF
5637             (mult:VF
5638               (vec_duplicate:VF
5639                 (match_operand:<VEL> 2 "register_operand"))
5640               (match_operand:VF 3 "register_operand"))
5641             (match_operand:VF 4 "register_operand"))
5642           (match_operand:VF 5 "register_operand")))]
5643   "TARGET_VECTOR"
5644 {})
5645
5646 (define_insn "*pred_<madd_msub><mode>_scalar"
5647   [(set (match_operand:VF 0 "register_operand"            "=vd, ?&vd, vr, ?&vr")
5648         (if_then_else:VF
5649           (unspec:<VM>
5650             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,   vm,Wc1,  Wc1")
5651              (match_operand 5 "vector_length_operand"     " rK,   rK, rK,   rK")
5652              (match_operand 6 "const_int_operand"         "  i,    i,  i,    i")
5653              (match_operand 7 "const_int_operand"         "  i,    i,  i,    i")
5654              (match_operand 8 "const_int_operand"         "  i,    i,  i,    i")
5655              (reg:SI VL_REGNUM)
5656              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5657           (plus_minus:VF
5658             (mult:VF
5659               (vec_duplicate:VF
5660                 (match_operand:<VEL> 2 "register_operand" "  f,  f,    f,    f"))
5661               (match_operand:VF 3 "register_operand"      "  0, vr,    0,   vr"))
5662             (match_operand:VF 4 "register_operand"        " vr, vr,   vr,   vr"))
5663           (match_dup 3)))]
5664   "TARGET_VECTOR"
5665   "@
5666    vf<madd_msub>.vf\t%0,%2,%4%p1
5667    vmv.v.v\t%0,%3\;vf<madd_msub>.vf\t%0,%2,%4%p1
5668    vf<madd_msub>.vf\t%0,%2,%4%p1
5669    vmv.v.v\t%0,%3\;vf<madd_msub>.vf\t%0,%2,%4%p1"
5670   [(set_attr "type" "vfmuladd")
5671    (set_attr "mode" "<MODE>")
5672    (set_attr "merge_op_idx" "4")
5673    (set_attr "vl_op_idx" "5")
5674    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5675    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5676    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5677
5678 (define_insn "*pred_<macc_msac><mode>_scalar"
5679   [(set (match_operand:VF 0 "register_operand"            "=vd, ?&vd, vr, ?&vr")
5680         (if_then_else:VF
5681           (unspec:<VM>
5682             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,   vm,Wc1,  Wc1")
5683              (match_operand 5 "vector_length_operand"     " rK,   rK, rK,   rK")
5684              (match_operand 6 "const_int_operand"         "  i,    i,  i,    i")
5685              (match_operand 7 "const_int_operand"         "  i,    i,  i,    i")
5686              (match_operand 8 "const_int_operand"         "  i,    i,  i,    i")
5687              (reg:SI VL_REGNUM)
5688              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5689           (plus_minus:VF
5690             (mult:VF
5691               (vec_duplicate:VF
5692                 (match_operand:<VEL> 2 "register_operand" "  f,  f,    f,    f"))
5693               (match_operand:VF 3 "register_operand"      " vr, vr,   vr,   vr"))
5694             (match_operand:VF 4 "register_operand"        "  0, vr,    0,   vr"))
5695           (match_dup 4)))]
5696   "TARGET_VECTOR"
5697   "@
5698    vf<macc_msac>.vf\t%0,%2,%3%p1
5699    vmv.v.v\t%0,%4\;vf<macc_msac>.vf\t%0,%2,%3%p1
5700    vf<macc_msac>.vf\t%0,%2,%3%p1
5701    vmv.v.v\t%0,%4\;vf<macc_msac>.vf\t%0,%2,%3%p1"
5702   [(set_attr "type" "vfmuladd")
5703    (set_attr "mode" "<MODE>")
5704    (set_attr "merge_op_idx" "2")
5705    (set_attr "vl_op_idx" "5")
5706    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5707    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5708    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5709
5710 (define_insn_and_rewrite "*pred_mul_<optab><mode>_scalar"
5711   [(set (match_operand:VF 0 "register_operand"            "=&vr, ?&vr")
5712         (if_then_else:VF
5713           (unspec:<VM>
5714             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1")
5715              (match_operand 6 "vector_length_operand"    "   rK,   rK")
5716              (match_operand 7 "const_int_operand"        "    i,    i")
5717              (match_operand 8 "const_int_operand"        "    i,    i")
5718              (match_operand 9 "const_int_operand"        "    i,    i")
5719              (reg:SI VL_REGNUM)
5720              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5721           (plus_minus:VF
5722             (mult:VF
5723               (vec_duplicate:VF
5724                 (match_operand:<VEL> 2 "register_operand" "    f,   f"))
5725               (match_operand:VF 3 "register_operand"      "   vr,  vr"))
5726             (match_operand:VF 4 "vector_arith_operand"    "   vr,  vr"))
5727           (match_operand:VF 5 "register_operand"          "    0,  vr")))]
5728   "TARGET_VECTOR
5729    && !rtx_equal_p (operands[3], operands[5])
5730    && !rtx_equal_p (operands[4], operands[5])"
5731   "@
5732    vmv.v.v\t%0,%4\;vf<macc_msac>.vf\t%0,%2,%3%p1
5733    #"
5734   "&& reload_completed
5735    && !rtx_equal_p (operands[0], operands[5])"
5736   {
5737     emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
5738                         operands[5], operands[4], operands[1], operands[6],
5739                         operands[7], operands[9]));
5740     operands[5] = operands[4] = operands[0];
5741   }
5742   [(set_attr "type" "vfmuladd")
5743    (set_attr "mode" "<MODE>")])
5744
5745 (define_expand "@pred_mul_neg_<optab><mode>"
5746   [(set (match_operand:VF 0 "register_operand")
5747         (if_then_else:VF
5748           (unspec:<VM>
5749             [(match_operand:<VM> 1 "vector_mask_operand")
5750              (match_operand 6 "vector_length_operand")
5751              (match_operand 7 "const_int_operand")
5752              (match_operand 8 "const_int_operand")
5753              (match_operand 9 "const_int_operand")
5754              (reg:SI VL_REGNUM)
5755              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5756           (plus_minus:VF
5757             (neg:VF
5758               (mult:VF
5759                 (match_operand:VF 2 "register_operand")
5760                 (match_operand:VF 3 "register_operand")))
5761             (match_operand:VF 4 "register_operand"))
5762           (match_operand:VF 5 "register_operand")))]
5763   "TARGET_VECTOR"
5764 {
5765   /* Swap the multiplication operands if the fallback value is the
5766      second of the two.  */
5767   if (rtx_equal_p (operands[3], operands[5]))
5768     std::swap (operands[2], operands[3]);
5769 })
5770
5771 (define_insn "*pred_<nmsub_nmadd><mode>"
5772   [(set (match_operand:VF 0 "register_operand"           "=vd, ?&vd, vr, ?&vr")
5773         (if_then_else:VF
5774           (unspec:<VM>
5775             [(match_operand:<VM> 1 "vector_mask_operand" " vm,   vm,Wc1,  Wc1")
5776              (match_operand 5 "vector_length_operand"    " rK,   rK, rK,   rK")
5777              (match_operand 6 "const_int_operand"        "  i,    i,  i,    i")
5778              (match_operand 7 "const_int_operand"        "  i,    i,  i,    i")
5779              (match_operand 8 "const_int_operand"        "  i,    i,  i,    i")
5780              (reg:SI VL_REGNUM)
5781              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5782           (plus_minus:VF
5783             (neg:VF
5784               (mult:VF
5785                 (match_operand:VF 2 "register_operand"   "  0,   vr,  0,   vr")
5786                 (match_operand:VF 3 "register_operand"   " vr,   vr, vr,   vr")))
5787             (match_operand:VF 4 "register_operand"       " vr,   vr, vr,   vr"))
5788           (match_dup 2)))]
5789   "TARGET_VECTOR"
5790   "@
5791    vf<nmsub_nmadd>.vv\t%0,%3,%4%p1
5792    vmv.v.v\t%0,%2\;vf<nmsub_nmadd>.vv\t%0,%3,%4%p1
5793    vf<nmsub_nmadd>.vv\t%0,%3,%4%p1
5794    vmv.v.v\t%0,%2\;vf<nmsub_nmadd>.vv\t%0,%3,%4%p1"
5795   [(set_attr "type" "vfmuladd")
5796    (set_attr "mode" "<MODE>")
5797    (set_attr "merge_op_idx" "4")
5798    (set_attr "vl_op_idx" "5")
5799    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5800    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5801    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5802
5803 (define_insn "*pred_<nmsac_nmacc><mode>"
5804   [(set (match_operand:VF 0 "register_operand"           "=vd, ?&vd, vr, ?&vr")
5805         (if_then_else:VF
5806           (unspec:<VM>
5807             [(match_operand:<VM> 1 "vector_mask_operand" " vm,   vm,Wc1,  Wc1")
5808              (match_operand 5 "vector_length_operand"    " rK,   rK, rK,   rK")
5809              (match_operand 6 "const_int_operand"        "  i,    i,  i,    i")
5810              (match_operand 7 "const_int_operand"        "  i,    i,  i,    i")
5811              (match_operand 8 "const_int_operand"        "  i,    i,  i,    i")
5812              (reg:SI VL_REGNUM)
5813              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5814           (plus_minus:VF
5815             (neg:VF
5816               (mult:VF
5817                 (match_operand:VF 2 "register_operand"   " vr,   vr, vr,   vr")
5818                 (match_operand:VF 3 "register_operand"   " vr,   vr, vr,   vr")))
5819             (match_operand:VF 4 "register_operand"       "  0,   vr,  0,   vr"))
5820           (match_dup 4)))]
5821   "TARGET_VECTOR"
5822   "@
5823    vf<nmsac_nmacc>.vv\t%0,%2,%3%p1
5824    vmv.v.v\t%0,%4\;vf<nmsac_nmacc>.vv\t%0,%2,%3%p1
5825    vf<nmsac_nmacc>.vv\t%0,%2,%3%p1
5826    vmv.v.v\t%0,%4\;vf<nmsac_nmacc>.vv\t%0,%2,%3%p1"
5827   [(set_attr "type" "vfmuladd")
5828    (set_attr "mode" "<MODE>")
5829    (set_attr "merge_op_idx" "2")
5830    (set_attr "vl_op_idx" "5")
5831    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5832    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5833    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5834
5835 (define_insn_and_rewrite "*pred_mul_neg_<optab><mode>"
5836   [(set (match_operand:VF 0 "register_operand"            "=&vr, ?&vr")
5837         (if_then_else:VF
5838           (unspec:<VM>
5839             [(match_operand:<VM> 1 "vector_mask_operand" "vmWc1,vmWc1")
5840              (match_operand 6 "vector_length_operand"    "   rK,   rK")
5841              (match_operand 7 "const_int_operand"        "    i,    i")
5842              (match_operand 8 "const_int_operand"        "    i,    i")
5843              (match_operand 9 "const_int_operand"        "    i,    i")
5844              (reg:SI VL_REGNUM)
5845              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5846           (plus_minus:VF
5847             (neg:VF
5848               (mult:VF
5849                 (match_operand:VF 2 "register_operand"     "   vr,   vr")
5850                 (match_operand:VF 3 "register_operand"     "   vr,   vr")))
5851             (match_operand:VF 4 "vector_arith_operand"   "   vr,   vr"))
5852           (match_operand:VF 5 "register_operand"         "    0,   vr")))]
5853   "TARGET_VECTOR
5854    && !rtx_equal_p (operands[2], operands[5])
5855    && !rtx_equal_p (operands[3], operands[5])
5856    && !rtx_equal_p (operands[4], operands[5])"
5857   "@
5858    vmv.v.v\t%0,%4\;vf<nmsac_nmacc>.vv\t%0,%2,%3%p1
5859    #"
5860   "&& reload_completed
5861    && !rtx_equal_p (operands[0], operands[5])"
5862   {
5863     emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
5864                         operands[5], operands[4], operands[1], operands[6],
5865                         operands[7], operands[9]));
5866     operands[5] = operands[4] = operands[0];
5867   }
5868   [(set_attr "type" "vfmuladd")
5869    (set_attr "mode" "<MODE>")])
5870
5871 (define_expand "@pred_mul_neg_<optab><mode>_scalar"
5872   [(set (match_operand:VF 0 "register_operand")
5873         (if_then_else:VF
5874           (unspec:<VM>
5875             [(match_operand:<VM> 1 "vector_mask_operand")
5876              (match_operand 6 "vector_length_operand")
5877              (match_operand 7 "const_int_operand")
5878              (match_operand 8 "const_int_operand")
5879              (match_operand 9 "const_int_operand")
5880              (reg:SI VL_REGNUM)
5881              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5882           (plus_minus:VF
5883             (neg:VF
5884               (mult:VF
5885                 (vec_duplicate:VF
5886                   (match_operand:<VEL> 2 "register_operand"))
5887                 (match_operand:VF 3 "register_operand")))
5888             (match_operand:VF 4 "register_operand"))
5889           (match_operand:VF 5 "register_operand")))]
5890   "TARGET_VECTOR"
5891 {})
5892
5893 (define_insn "*pred_<nmsub_nmadd><mode>_scalar"
5894   [(set (match_operand:VF 0 "register_operand"            "=vd, ?&vd, vr, ?&vr")
5895         (if_then_else:VF
5896           (unspec:<VM>
5897             [(match_operand:<VM> 1 "vector_mask_operand"  " vm,   vm,Wc1,  Wc1")
5898              (match_operand 5 "vector_length_operand"     " rK,   rK, rK,   rK")
5899              (match_operand 6 "const_int_operand"         "  i,    i,  i,    i")
5900              (match_operand 7 "const_int_operand"         "  i,    i,  i,    i")
5901              (match_operand 8 "const_int_operand"         "  i,    i,  i,    i")
5902              (reg:SI VL_REGNUM)
5903              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5904           (plus_minus:VF
5905             (neg:VF
5906               (mult:VF
5907                 (vec_duplicate:VF
5908                   (match_operand:<VEL> 2 "register_operand" "  f,    f,  f,    f"))
5909                 (match_operand:VF 3 "register_operand"      "  0,   vr,  0,   vr")))
5910             (match_operand:VF 4 "register_operand"          " vr,   vr, vr,   vr"))
5911           (match_dup 3)))]
5912   "TARGET_VECTOR"
5913   "@
5914    vf<nmsub_nmadd>.vf\t%0,%2,%4%p1
5915    vmv.v.v\t%0,%3\;vf<nmsub_nmadd>.vf\t%0,%2,%4%p1
5916    vf<nmsub_nmadd>.vf\t%0,%2,%4%p1
5917    vmv.v.v\t%0,%3\;vf<nmsub_nmadd>.vf\t%0,%2,%4%p1"
5918   [(set_attr "type" "vfmuladd")
5919    (set_attr "mode" "<MODE>")
5920    (set_attr "merge_op_idx" "4")
5921    (set_attr "vl_op_idx" "5")
5922    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5923    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5924    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5925
5926 (define_insn "*pred_<nmsac_nmacc><mode>_scalar"
5927   [(set (match_operand:VF 0 "register_operand"              "=vd, ?&vd, vr, ?&vr")
5928         (if_then_else:VF
5929           (unspec:<VM>
5930             [(match_operand:<VM> 1 "vector_mask_operand"    " vm,   vm,Wc1,  Wc1")
5931              (match_operand 5 "vector_length_operand"       " rK,   rK, rK,   rK")
5932              (match_operand 6 "const_int_operand"           "  i,    i,  i,    i")
5933              (match_operand 7 "const_int_operand"           "  i,    i,  i,    i")
5934              (match_operand 8 "const_int_operand"           "  i,    i,  i,    i")
5935              (reg:SI VL_REGNUM)
5936              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5937           (plus_minus:VF
5938             (neg:VF
5939               (mult:VF
5940                 (vec_duplicate:VF
5941                   (match_operand:<VEL> 2 "register_operand" "  f,    f,  f,    f"))
5942                 (match_operand:VF 3 "register_operand"      " vr,   vr, vr,   vr")))
5943             (match_operand:VF 4 "register_operand"          "  0,   vr,  0,   vr"))
5944           (match_dup 4)))]
5945   "TARGET_VECTOR"
5946   "@
5947    vf<nmsac_nmacc>.vf\t%0,%2,%3%p1
5948    vmv.v.v\t%0,%4\;vf<nmsac_nmacc>.vf\t%0,%2,%3%p1
5949    vf<nmsac_nmacc>.vf\t%0,%2,%3%p1
5950    vmv.v.v\t%0,%4\;vf<nmsac_nmacc>.vf\t%0,%2,%3%p1"
5951   [(set_attr "type" "vfmuladd")
5952    (set_attr "mode" "<MODE>")
5953    (set_attr "merge_op_idx" "2")
5954    (set_attr "vl_op_idx" "5")
5955    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])"))
5956    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])"))
5957    (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))])
5958
5959 (define_insn_and_rewrite "*pred_mul_neg_<optab><mode>_scalar"
5960   [(set (match_operand:VF 0 "register_operand"               "=&vr, ?&vr")
5961         (if_then_else:VF
5962           (unspec:<VM>
5963             [(match_operand:<VM> 1 "vector_mask_operand"    "vmWc1,vmWc1")
5964              (match_operand 6 "vector_length_operand"       "   rK,   rK")
5965              (match_operand 7 "const_int_operand"           "    i,    i")
5966              (match_operand 8 "const_int_operand"           "    i,    i")
5967              (match_operand 9 "const_int_operand"           "    i,    i")
5968              (reg:SI VL_REGNUM)
5969              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
5970           (plus_minus:VF
5971             (neg:VF
5972               (mult:VF
5973                 (vec_duplicate:VF
5974                   (match_operand:<VEL> 2 "register_operand" "    f,   f"))
5975                 (match_operand:VF 3 "register_operand"      "   vr,  vr")))
5976             (match_operand:VF 4 "vector_arith_operand"      "   vr,  vr"))
5977           (match_operand:VF 5 "register_operand"            "    0,  vr")))]
5978   "TARGET_VECTOR
5979    && !rtx_equal_p (operands[3], operands[5])
5980    && !rtx_equal_p (operands[4], operands[5])"
5981   "@
5982    vmv.v.v\t%0,%4\;vf<nmsac_nmacc>.vf\t%0,%2,%3%p1
5983    #"
5984   "&& reload_completed
5985    && !rtx_equal_p (operands[0], operands[5])"
5986   {
5987     emit_insn (gen_pred_merge<mode> (operands[0], RVV_VUNDEF (<MODE>mode),
5988                         operands[5], operands[4], operands[1], operands[6],
5989                         operands[7], operands[9]));
5990     operands[5] = operands[4] = operands[0];
5991   }
5992   [(set_attr "type" "vfmuladd")
5993    (set_attr "mode" "<MODE>")])
5994
5995 ;; -------------------------------------------------------------------------------
5996 ;; ---- Predicated floating-point unary operations
5997 ;; -------------------------------------------------------------------------------
5998 ;; Includes:
5999 ;; - 13.8 Vector Floating-Point Square-Root Instruction
6000 ;; - 13.9 Vector Floating-Point Reciprocal Square-Root Estimate Instruction
6001 ;; - 13.10 Vector Floating-Point Reciprocal Estimate Instruction
6002 ;; - 13.12 Vector Floating-Point Sign-Injection Instructions (vfneg.v/vfabs.v)
6003 ;; - 13.14 Vector Floating-Point Classify Instruction
6004 ;; -------------------------------------------------------------------------------
6005
6006 (define_insn "@pred_<optab><mode>"
6007   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
6008         (if_then_else:VF
6009           (unspec:<VM>
6010             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
6011              (match_operand 4 "vector_length_operand"    " rK, rK, rK, rK")
6012              (match_operand 5 "const_int_operand"        "  i,  i,  i,  i")
6013              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
6014              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
6015              (reg:SI VL_REGNUM)
6016              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6017           (any_float_unop:VF
6018             (match_operand:VF 3 "register_operand"       " vr, vr, vr, vr"))
6019           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
6020   "TARGET_VECTOR"
6021   "vf<insn>.v\t%0,%3%p1"
6022   [(set_attr "type" "<float_insn_type>")
6023    (set_attr "mode" "<MODE>")
6024    (set_attr "vl_op_idx" "4")
6025    (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[5])"))
6026    (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[6])"))
6027    (set (attr "avl_type") (symbol_ref "INTVAL (operands[7])"))])
6028
6029 (define_insn "@pred_<misc_op><mode>"
6030   [(set (match_operand:VF 0 "register_operand"           "=vd, vd, vr, vr")
6031         (if_then_else:VF
6032           (unspec:<VM>
6033             [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
6034              (match_operand 4 "vector_length_operand"    " rK, rK, rK, rK")
6035              (match_operand 5 "const_int_operand"        "  i,  i,  i,  i")
6036              (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
6037              (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
6038              (reg:SI VL_REGNUM)
6039              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6040           (unspec:VF
6041             [(match_operand:VF 3 "register_operand"       " vr, vr, vr, vr")] VFMISC)
6042           (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")))]
6043   "TARGET_VECTOR"
6044   "vf<misc_op>.v\t%0,%3%p1"
6045   [(set_attr "type" "<float_insn_type>")
6046    (set_attr "mode" "<MODE>")])
6047
6048 (define_insn "@pred_class<mode>"
6049   [(set (match_operand:<VCONVERT> 0 "register_operand"       "=vd, vd, vr, vr")
6050         (if_then_else:<VCONVERT>
6051           (unspec:<VM>
6052             [(match_operand:<VM> 1 "vector_mask_operand"     " vm, vm,Wc1,Wc1")
6053              (match_operand 4 "vector_length_operand"        " rK, rK, rK, rK")
6054              (match_operand 5 "const_int_operand"            "  i,  i,  i,  i")
6055              (match_operand 6 "const_int_operand"            "  i,  i,  i,  i")
6056              (match_operand 7 "const_int_operand"            "  i,  i,  i,  i")
6057              (reg:SI VL_REGNUM)
6058              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6059           (unspec:<VCONVERT>
6060             [(match_operand:VF 3 "register_operand"          " vr, vr, vr, vr")] UNSPEC_VFCLASS)
6061           (match_operand:<VCONVERT> 2 "vector_merge_operand" " vu,  0, vu,  0")))]
6062   "TARGET_VECTOR"
6063   "vfclass.v\t%0,%3%p1"
6064   [(set_attr "type" "vfclass")
6065    (set_attr "mode" "<MODE>")])
6066
6067 ;; -------------------------------------------------------------------------------
6068 ;; ---- Predicated floating-point widen binary operations
6069 ;; -------------------------------------------------------------------------------
6070 ;; Includes:
6071 ;; - 13.3 Vector Widening Floating-Point Add/Subtract Instructions
6072 ;; - 13.5 Vector Widening Floating-Point Multiply
6073 ;; -------------------------------------------------------------------------------
6074
6075 ;; Vector Widening Add/Subtract/Multiply.
6076 (define_insn "@pred_dual_widen_<optab><mode>"
6077   [(set (match_operand:VWEXTF 0 "register_operand"                  "=&vr,  &vr")
6078         (if_then_else:VWEXTF
6079           (unspec:<VM>
6080             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
6081              (match_operand 5 "vector_length_operand"              "   rK,   rK")
6082              (match_operand 6 "const_int_operand"                  "    i,    i")
6083              (match_operand 7 "const_int_operand"                  "    i,    i")
6084              (match_operand 8 "const_int_operand"                  "    i,    i")
6085              (reg:SI VL_REGNUM)
6086              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6087           (any_widen_binop:VWEXTF
6088             (float_extend:VWEXTF
6089               (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
6090             (float_extend:VWEXTF
6091               (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr,   vr")))
6092           (match_operand:VWEXTF 2 "vector_merge_operand"           "   vu,    0")))]
6093   "TARGET_VECTOR"
6094   "vfw<insn>.vv\t%0,%3,%4%p1"
6095   [(set_attr "type" "vf<widen_binop_insn_type>")
6096    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6097
6098 (define_insn "@pred_dual_widen_<optab><mode>_scalar"
6099   [(set (match_operand:VWEXTF 0 "register_operand"                  "=&vr,  &vr")
6100         (if_then_else:VWEXTF
6101           (unspec:<VM>
6102             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
6103              (match_operand 5 "vector_length_operand"              "   rK,   rK")
6104              (match_operand 6 "const_int_operand"                  "    i,    i")
6105              (match_operand 7 "const_int_operand"                  "    i,    i")
6106              (match_operand 8 "const_int_operand"                  "    i,    i")
6107              (reg:SI VL_REGNUM)
6108              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6109           (any_widen_binop:VWEXTF
6110             (float_extend:VWEXTF
6111               (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
6112             (float_extend:VWEXTF
6113               (vec_duplicate:<V_DOUBLE_TRUNC>
6114                 (match_operand:<VSUBEL> 4 "register_operand"       "    f,    f"))))
6115           (match_operand:VWEXTF 2 "vector_merge_operand"           "   vu,    0")))]
6116   "TARGET_VECTOR"
6117   "vfw<insn>.vf\t%0,%3,%4%p1"
6118   [(set_attr "type" "vf<widen_binop_insn_type>")
6119    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6120
6121 (define_insn "@pred_single_widen_<plus_minus:optab><mode>"
6122   [(set (match_operand:VWEXTF 0 "register_operand"                  "=&vr,  &vr")
6123         (if_then_else:VWEXTF
6124           (unspec:<VM>
6125             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
6126              (match_operand 5 "vector_length_operand"              "   rK,   rK")
6127              (match_operand 6 "const_int_operand"                  "    i,    i")
6128              (match_operand 7 "const_int_operand"                  "    i,    i")
6129              (match_operand 8 "const_int_operand"                  "    i,    i")
6130              (reg:SI VL_REGNUM)
6131              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6132           (plus_minus:VWEXTF
6133             (match_operand:VWEXTF 3 "register_operand"             "   vr,   vr")
6134             (float_extend:VWEXTF
6135               (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr,   vr")))
6136           (match_operand:VWEXTF 2 "vector_merge_operand"           "   vu,    0")))]
6137   "TARGET_VECTOR"
6138   "vfw<insn>.wv\t%0,%3,%4%p1"
6139   [(set_attr "type" "vf<widen_binop_insn_type>")
6140    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6141
6142 (define_insn "@pred_single_widen_<plus_minus:optab><mode>_scalar"
6143   [(set (match_operand:VWEXTF 0 "register_operand"                  "=&vr,  &vr")
6144         (if_then_else:VWEXTF
6145           (unspec:<VM>
6146             [(match_operand:<VM> 1 "vector_mask_operand"           "vmWc1,vmWc1")
6147              (match_operand 5 "vector_length_operand"              "   rK,   rK")
6148              (match_operand 6 "const_int_operand"                  "    i,    i")
6149              (match_operand 7 "const_int_operand"                  "    i,    i")
6150              (match_operand 8 "const_int_operand"                  "    i,    i")
6151              (reg:SI VL_REGNUM)
6152              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6153           (plus_minus:VWEXTF
6154             (match_operand:VWEXTF 3 "register_operand"             "   vr,   vr")
6155             (float_extend:VWEXTF
6156               (vec_duplicate:<V_DOUBLE_TRUNC>
6157                 (match_operand:<VSUBEL> 4 "register_operand"       "    f,    f"))))
6158           (match_operand:VWEXTF 2 "vector_merge_operand"           "   vu,    0")))]
6159   "TARGET_VECTOR"
6160   "vfw<insn>.wf\t%0,%3,%4%p1"
6161   [(set_attr "type" "vf<widen_binop_insn_type>")
6162    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6163
6164 ;; -------------------------------------------------------------------------------
6165 ;; ---- Predicated widen floating-point ternary operations
6166 ;; -------------------------------------------------------------------------------
6167 ;; Includes:
6168 ;; - 13.7 Vector Widening Floating-Point Fused Multiply-Add Instructions
6169 ;; -------------------------------------------------------------------------------
6170
6171 (define_insn "@pred_widen_mul_<optab><mode>"
6172   [(set (match_operand:VWEXTF 0 "register_operand"                    "=&vr")
6173         (if_then_else:VWEXTF
6174           (unspec:<VM>
6175             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1")
6176              (match_operand 5 "vector_length_operand"                "   rK")
6177              (match_operand 6 "const_int_operand"                    "    i")
6178              (match_operand 7 "const_int_operand"                    "    i")
6179              (match_operand 8 "const_int_operand"                    "    i")
6180              (reg:SI VL_REGNUM)
6181              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6182           (plus_minus:VWEXTF
6183             (mult:VWEXTF
6184               (float_extend:VWEXTF
6185                 (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr"))
6186               (float_extend:VWEXTF
6187                 (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
6188             (match_operand:VWEXTF 2 "register_operand"               "    0"))
6189           (match_dup 2)))]
6190   "TARGET_VECTOR"
6191   "vfw<macc_msac>.vv\t%0,%3,%4%p1"
6192   [(set_attr "type" "vfwmuladd")
6193    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6194
6195 (define_insn "@pred_widen_mul_<optab><mode>_scalar"
6196   [(set (match_operand:VWEXTF 0 "register_operand"                    "=&vr")
6197         (if_then_else:VWEXTF
6198           (unspec:<VM>
6199             [(match_operand:<VM> 1 "vector_mask_operand"             "vmWc1")
6200              (match_operand 5 "vector_length_operand"                "   rK")
6201              (match_operand 6 "const_int_operand"                    "    i")
6202              (match_operand 7 "const_int_operand"                    "    i")
6203              (match_operand 8 "const_int_operand"                    "    i")
6204              (reg:SI VL_REGNUM)
6205              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6206           (plus_minus:VWEXTF
6207             (mult:VWEXTF
6208               (float_extend:VWEXTF
6209                 (vec_duplicate:<V_DOUBLE_TRUNC>
6210                   (match_operand:<VSUBEL> 3 "register_operand"       "    f")))
6211               (float_extend:VWEXTF
6212                 (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
6213             (match_operand:VWEXTF 2 "register_operand"               "    0"))
6214           (match_dup 2)))]
6215   "TARGET_VECTOR"
6216   "vfw<macc_msac>.vf\t%0,%3,%4%p1"
6217   [(set_attr "type" "vfwmuladd")
6218    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6219
6220 (define_insn "@pred_widen_mul_neg_<optab><mode>"
6221   [(set (match_operand:VWEXTF 0 "register_operand"                      "=&vr")
6222         (if_then_else:VWEXTF
6223           (unspec:<VM>
6224             [(match_operand:<VM> 1 "vector_mask_operand"               "vmWc1")
6225              (match_operand 5 "vector_length_operand"                  "   rK")
6226              (match_operand 6 "const_int_operand"                      "    i")
6227              (match_operand 7 "const_int_operand"                      "    i")
6228              (match_operand 8 "const_int_operand"                      "    i")
6229              (reg:SI VL_REGNUM)
6230              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6231           (plus_minus:VWEXTF
6232             (neg:VWEXTF
6233               (mult:VWEXTF
6234                 (float_extend:VWEXTF
6235                   (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr"))
6236                 (float_extend:VWEXTF
6237                   (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr"))))
6238               (match_operand:VWEXTF 2 "register_operand"               "    0"))
6239           (match_dup 2)))]
6240   "TARGET_VECTOR"
6241   "vfw<nmsac_nmacc>.vv\t%0,%3,%4%p1"
6242   [(set_attr "type" "vfwmuladd")
6243    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6244
6245 (define_insn "@pred_widen_mul_neg_<optab><mode>_scalar"
6246   [(set (match_operand:VWEXTF 0 "register_operand"                      "=&vr")
6247         (if_then_else:VWEXTF
6248           (unspec:<VM>
6249             [(match_operand:<VM> 1 "vector_mask_operand"               "vmWc1")
6250              (match_operand 5 "vector_length_operand"                  "   rK")
6251              (match_operand 6 "const_int_operand"                      "    i")
6252              (match_operand 7 "const_int_operand"                      "    i")
6253              (match_operand 8 "const_int_operand"                      "    i")
6254              (reg:SI VL_REGNUM)
6255              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6256           (plus_minus:VWEXTF
6257             (neg:VWEXTF
6258               (mult:VWEXTF
6259                 (float_extend:VWEXTF
6260                   (vec_duplicate:<V_DOUBLE_TRUNC>
6261                     (match_operand:<VSUBEL> 3 "register_operand"       "    f")))
6262                 (float_extend:VWEXTF
6263                   (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr"))))
6264             (match_operand:VWEXTF 2 "register_operand"                 "    0"))
6265           (match_dup 2)))]
6266   "TARGET_VECTOR"
6267   "vfw<nmsac_nmacc>.vf\t%0,%3,%4%p1"
6268   [(set_attr "type" "vfwmuladd")
6269    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6270
6271 ;; -------------------------------------------------------------------------------
6272 ;; ---- Predicated floating-point comparison operations
6273 ;; -------------------------------------------------------------------------------
6274 ;; Includes:
6275 ;; - 13.13 Vector Floating-Point Compare Instructions
6276 ;; -------------------------------------------------------------------------------
6277
6278 (define_expand "@pred_cmp<mode>"
6279   [(set (match_operand:<VM> 0 "register_operand")
6280         (if_then_else:<VM>
6281           (unspec:<VM>
6282             [(match_operand:<VM> 1 "vector_mask_operand")
6283              (match_operand 6 "vector_length_operand")
6284              (match_operand 7 "const_int_operand")
6285              (match_operand 8 "const_int_operand")
6286              (reg:SI VL_REGNUM)
6287              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6288           (match_operator:<VM> 3 "signed_order_operator"
6289              [(match_operand:VF 4 "register_operand")
6290               (match_operand:VF 5 "register_operand")])
6291           (match_operand:<VM> 2 "vector_merge_operand")))]
6292   "TARGET_VECTOR"
6293   {})
6294
6295 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
6296 (define_insn "*pred_cmp<mode>"
6297   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr")
6298         (if_then_else:<VM>
6299           (unspec:<VM>
6300             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6301              (match_operand 6 "vector_length_operand"         "   rK,   rK")
6302              (match_operand 7 "const_int_operand"             "    i,    i")
6303              (match_operand 8 "const_int_operand"             "    i,    i")
6304              (reg:SI VL_REGNUM)
6305              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6306           (match_operator:<VM> 3 "signed_order_operator"
6307              [(match_operand:VF 4 "register_operand"          "   vr,   vr")
6308               (match_operand:VF 5 "register_operand"          "   vr,   vr")])
6309           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
6310   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
6311   "vmf%B3.vv\t%0,%4,%5%p1"
6312   [(set_attr "type" "vfcmp")
6313    (set_attr "mode" "<MODE>")])
6314
6315 ;; We use early-clobber for source LMUL > dest LMUL.
6316 (define_insn "*pred_cmp<mode>_narrow"
6317   [(set (match_operand:<VM> 0 "register_operand"               "=&vr,  &vr")
6318         (if_then_else:<VM>
6319           (unspec:<VM>
6320             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6321              (match_operand 6 "vector_length_operand"         "   rK,   rK")
6322              (match_operand 7 "const_int_operand"             "    i,    i")
6323              (match_operand 8 "const_int_operand"             "    i,    i")
6324              (reg:SI VL_REGNUM)
6325              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6326           (match_operator:<VM> 3 "signed_order_operator"
6327              [(match_operand:VF 4 "register_operand"          "   vr,   vr")
6328               (match_operand:VF 5 "register_operand"          "   vr,   vr")])
6329           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
6330   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
6331   "vmf%B3.vv\t%0,%4,%5%p1"
6332   [(set_attr "type" "vfcmp")
6333    (set_attr "mode" "<MODE>")])
6334
6335 (define_expand "@pred_cmp<mode>_scalar"
6336   [(set (match_operand:<VM> 0 "register_operand")
6337         (if_then_else:<VM>
6338           (unspec:<VM>
6339             [(match_operand:<VM> 1 "vector_mask_operand")
6340              (match_operand 6 "vector_length_operand")
6341              (match_operand 7 "const_int_operand")
6342              (match_operand 8 "const_int_operand")
6343              (reg:SI VL_REGNUM)
6344              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6345           (match_operator:<VM> 3 "signed_order_operator"
6346              [(match_operand:VF 4 "register_operand")
6347               (vec_duplicate:VF
6348                 (match_operand:<VEL> 5 "register_operand"))])
6349           (match_operand:<VM> 2 "vector_merge_operand")))]
6350   "TARGET_VECTOR"
6351   {})
6352
6353 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
6354 (define_insn "*pred_cmp<mode>_scalar"
6355   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr")
6356         (if_then_else:<VM>
6357           (unspec:<VM>
6358             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6359              (match_operand 6 "vector_length_operand"         "   rK,   rK")
6360              (match_operand 7 "const_int_operand"             "    i,    i")
6361              (match_operand 8 "const_int_operand"             "    i,    i")
6362              (reg:SI VL_REGNUM)
6363              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6364           (match_operator:<VM> 3 "signed_order_operator"
6365              [(match_operand:VF 4 "register_operand"          "   vr,   vr")
6366               (vec_duplicate:VF
6367                 (match_operand:<VEL> 5 "register_operand"     "    f,    f"))])
6368           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
6369   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
6370   "vmf%B3.vf\t%0,%4,%5%p1"
6371   [(set_attr "type" "vfcmp")
6372    (set_attr "mode" "<MODE>")])
6373
6374 ;; We use early-clobber for source LMUL > dest LMUL.
6375 (define_insn "*pred_cmp<mode>_scalar_narrow"
6376   [(set (match_operand:<VM> 0 "register_operand"               "=&vr,  &vr")
6377         (if_then_else:<VM>
6378           (unspec:<VM>
6379             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6380              (match_operand 6 "vector_length_operand"         "   rK,   rK")
6381              (match_operand 7 "const_int_operand"             "    i,    i")
6382              (match_operand 8 "const_int_operand"             "    i,    i")
6383              (reg:SI VL_REGNUM)
6384              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6385           (match_operator:<VM> 3 "signed_order_operator"
6386              [(match_operand:VF 4 "register_operand"          "   vr,   vr")
6387               (vec_duplicate:VF
6388                 (match_operand:<VEL> 5 "register_operand"     "    f,    f"))])
6389           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
6390   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
6391   "vmf%B3.vf\t%0,%4,%5%p1"
6392   [(set_attr "type" "vfcmp")
6393    (set_attr "mode" "<MODE>")])
6394
6395 (define_expand "@pred_eqne<mode>_scalar"
6396   [(set (match_operand:<VM> 0 "register_operand")
6397         (if_then_else:<VM>
6398           (unspec:<VM>
6399             [(match_operand:<VM> 1 "vector_mask_operand")
6400              (match_operand 6 "vector_length_operand")
6401              (match_operand 7 "const_int_operand")
6402              (match_operand 8 "const_int_operand")
6403              (reg:SI VL_REGNUM)
6404              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6405           (match_operator:<VM> 3 "equality_operator"
6406              [(vec_duplicate:VF
6407                 (match_operand:<VEL> 5 "register_operand"))
6408               (match_operand:VF 4 "register_operand")])
6409           (match_operand:<VM> 2 "vector_merge_operand")))]
6410   "TARGET_VECTOR"
6411   {})
6412
6413 ;; We don't use early-clobber for LMUL <= 1 to get better codegen.
6414 (define_insn "*pred_eqne<mode>_scalar"
6415   [(set (match_operand:<VM> 0 "register_operand"                "=vr,   vr")
6416         (if_then_else:<VM>
6417           (unspec:<VM>
6418             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6419              (match_operand 6 "vector_length_operand"         "   rK,   rK")
6420              (match_operand 7 "const_int_operand"             "    i,    i")
6421              (match_operand 8 "const_int_operand"             "    i,    i")
6422              (reg:SI VL_REGNUM)
6423              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6424           (match_operator:<VM> 3 "equality_operator"
6425              [(vec_duplicate:VF
6426                 (match_operand:<VEL> 5 "register_operand"     "    f,    f"))
6427               (match_operand:VF 4 "register_operand"          "   vr,   vr")])
6428           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
6429   "TARGET_VECTOR && known_le (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
6430   "vmf%B3.vf\t%0,%4,%5%p1"
6431   [(set_attr "type" "vfcmp")
6432    (set_attr "mode" "<MODE>")])
6433
6434 ;; We use early-clobber for source LMUL > dest LMUL.
6435 (define_insn "*pred_eqne<mode>_scalar_narrow"
6436   [(set (match_operand:<VM> 0 "register_operand"               "=&vr,  &vr")
6437         (if_then_else:<VM>
6438           (unspec:<VM>
6439             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6440              (match_operand 6 "vector_length_operand"         "   rK,   rK")
6441              (match_operand 7 "const_int_operand"             "    i,    i")
6442              (match_operand 8 "const_int_operand"             "    i,    i")
6443              (reg:SI VL_REGNUM)
6444              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6445           (match_operator:<VM> 3 "equality_operator"
6446              [(vec_duplicate:VF
6447                 (match_operand:<VEL> 5 "register_operand"     "    f,    f"))
6448               (match_operand:VF 4 "register_operand"          "   vr,   vr")])
6449           (match_operand:<VM> 2 "vector_merge_operand"        "   vu,    0")))]
6450   "TARGET_VECTOR && known_gt (GET_MODE_SIZE (<MODE>mode), BYTES_PER_RISCV_VECTOR)"
6451   "vmf%B3.vf\t%0,%4,%5%p1"
6452   [(set_attr "type" "vfcmp")
6453    (set_attr "mode" "<MODE>")])
6454
6455 ;; -------------------------------------------------------------------------------
6456 ;; ---- Predicated floating-point merge
6457 ;; -------------------------------------------------------------------------------
6458 ;; Includes:
6459 ;; - 13.15 Vector Floating-Point Merge Instruction
6460 ;; -------------------------------------------------------------------------------
6461
6462 (define_insn "@pred_merge<mode>_scalar"
6463   [(set (match_operand:VF 0 "register_operand"      "=vd,vd")
6464     (if_then_else:VF
6465       (unspec:<VM>
6466         [(match_operand 5 "vector_length_operand"   " rK,rK")
6467          (match_operand 6 "const_int_operand"       "  i, i")
6468          (match_operand 7 "const_int_operand"       "  i, i")
6469          (reg:SI VL_REGNUM)
6470          (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6471       (vec_merge:VF
6472         (vec_duplicate:VF
6473           (match_operand:<VEL> 3 "register_operand" "  f, f"))
6474         (match_operand:VF 2 "register_operand"      " vr,vr")
6475         (match_operand:<VM> 4 "register_operand"    " vm,vm"))
6476       (match_operand:VF 1 "vector_merge_operand"    " vu, 0")))]
6477   "TARGET_VECTOR"
6478   "vfmerge.vfm\t%0,%2,%3,%4"
6479   [(set_attr "type" "vfmerge")
6480    (set_attr "mode" "<MODE>")])
6481
6482 ;; -------------------------------------------------------------------------------
6483 ;; ---- Predicated floating-point conversions
6484 ;; -------------------------------------------------------------------------------
6485 ;; Includes:
6486 ;; - 13.17 Single-Width Floating-Point/Integer Type-Convert Instructions
6487 ;; -------------------------------------------------------------------------------
6488
6489 (define_insn "@pred_fcvt_x<v_su>_f<mode>"
6490   [(set (match_operand:<VCONVERT> 0 "register_operand"       "=vd, vd, vr, vr")
6491         (if_then_else:<VCONVERT>
6492           (unspec:<VM>
6493             [(match_operand:<VM> 1 "vector_mask_operand"     " vm, vm,Wc1,Wc1")
6494              (match_operand 4 "vector_length_operand"        " rK, rK, rK, rK")
6495              (match_operand 5 "const_int_operand"            "  i,  i,  i,  i")
6496              (match_operand 6 "const_int_operand"            "  i,  i,  i,  i")
6497              (match_operand 7 "const_int_operand"            "  i,  i,  i,  i")
6498              (reg:SI VL_REGNUM)
6499              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6500           (unspec:<VCONVERT>
6501              [(match_operand:VF 3 "register_operand"         " vr, vr, vr, vr")] VFCVTS)
6502           (match_operand:<VCONVERT> 2 "vector_merge_operand" " vu,  0, vu,  0")))]
6503   "TARGET_VECTOR"
6504   "vfcvt.x<v_su>.f.v\t%0,%3%p1"
6505   [(set_attr "type" "vfcvtftoi")
6506    (set_attr "mode" "<MODE>")])
6507
6508 (define_insn "@pred_<fix_cvt><mode>"
6509   [(set (match_operand:<VCONVERT> 0 "register_operand"       "=vd, vd, vr, vr")
6510         (if_then_else:<VCONVERT>
6511           (unspec:<VM>
6512             [(match_operand:<VM> 1 "vector_mask_operand"     " vm, vm,Wc1,Wc1")
6513              (match_operand 4 "vector_length_operand"        " rK, rK, rK, rK")
6514              (match_operand 5 "const_int_operand"            "  i,  i,  i,  i")
6515              (match_operand 6 "const_int_operand"            "  i,  i,  i,  i")
6516              (match_operand 7 "const_int_operand"            "  i,  i,  i,  i")
6517              (reg:SI VL_REGNUM)
6518              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6519           (any_fix:<VCONVERT>
6520              (match_operand:VF 3 "register_operand"          " vr, vr, vr, vr"))
6521           (match_operand:<VCONVERT> 2 "vector_merge_operand" " vu,  0, vu,  0")))]
6522   "TARGET_VECTOR"
6523   "vfcvt.rtz.x<u>.f.v\t%0,%3%p1"
6524   [(set_attr "type" "vfcvtftoi")
6525    (set_attr "mode" "<MODE>")])
6526
6527 (define_insn "@pred_<float_cvt><mode>"
6528   [(set (match_operand:VF 0 "register_operand"              "=vd, vd, vr, vr")
6529         (if_then_else:VF
6530           (unspec:<VM>
6531             [(match_operand:<VM> 1 "vector_mask_operand"    " vm, vm,Wc1,Wc1")
6532              (match_operand 4 "vector_length_operand"       " rK, rK, rK, rK")
6533              (match_operand 5 "const_int_operand"           "  i,  i,  i,  i")
6534              (match_operand 6 "const_int_operand"           "  i,  i,  i,  i")
6535              (match_operand 7 "const_int_operand"           "  i,  i,  i,  i")
6536              (reg:SI VL_REGNUM)
6537              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6538           (any_float:VF
6539              (match_operand:<VCONVERT> 3 "register_operand" " vr, vr, vr, vr"))
6540           (match_operand:VF 2 "vector_merge_operand"        " vu,  0, vu,  0")))]
6541   "TARGET_VECTOR"
6542   "vfcvt.f.x<u>.v\t%0,%3%p1"
6543   [(set_attr "type" "vfcvtitof")
6544    (set_attr "mode" "<MODE>")])
6545
6546 ;; -------------------------------------------------------------------------------
6547 ;; ---- Predicated floating-point widen conversions
6548 ;; -------------------------------------------------------------------------------
6549 ;; Includes:
6550 ;; - 13.18 Widening Floating-Point/Integer Type-Convert Instructions
6551 ;; -------------------------------------------------------------------------------
6552
6553 (define_insn "@pred_widen_fcvt_x<v_su>_f<mode>"
6554   [(set (match_operand:VWCONVERTI 0 "register_operand"         "=&vr,  &vr")
6555         (if_then_else:VWCONVERTI
6556           (unspec:<VM>
6557             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6558              (match_operand 4 "vector_length_operand"         "   rK,   rK")
6559              (match_operand 5 "const_int_operand"             "    i,    i")
6560              (match_operand 6 "const_int_operand"             "    i,    i")
6561              (match_operand 7 "const_int_operand"             "    i,    i")
6562              (reg:SI VL_REGNUM)
6563              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6564           (unspec:VWCONVERTI
6565              [(match_operand:<VNCONVERT> 3 "register_operand" "   vr,   vr")] VFCVTS)
6566           (match_operand:VWCONVERTI 2 "vector_merge_operand"  "   vu,    0")))]
6567   "TARGET_VECTOR"
6568   "vfwcvt.x<v_su>.f.v\t%0,%3%p1"
6569   [(set_attr "type" "vfwcvtftoi")
6570    (set_attr "mode" "<VNCONVERT>")])
6571
6572 (define_insn "@pred_widen_<fix_cvt><mode>"
6573   [(set (match_operand:VWCONVERTI 0 "register_operand"        "=&vr,  &vr")
6574         (if_then_else:VWCONVERTI
6575           (unspec:<VM>
6576             [(match_operand:<VM> 1 "vector_mask_operand"     "vmWc1,vmWc1")
6577              (match_operand 4 "vector_length_operand"        "   rK,   rK")
6578              (match_operand 5 "const_int_operand"            "    i,    i")
6579              (match_operand 6 "const_int_operand"            "    i,    i")
6580              (match_operand 7 "const_int_operand"            "    i,    i")
6581              (reg:SI VL_REGNUM)
6582              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6583           (any_fix:VWCONVERTI
6584              (match_operand:<VNCONVERT> 3 "register_operand" "   vr,   vr"))
6585           (match_operand:VWCONVERTI 2 "vector_merge_operand" "   vu,    0")))]
6586   "TARGET_VECTOR"
6587   "vfwcvt.rtz.x<u>.f.v\t%0,%3%p1"
6588   [(set_attr "type" "vfwcvtftoi")
6589    (set_attr "mode" "<VNCONVERT>")])
6590
6591 (define_insn "@pred_widen_<float_cvt><mode>"
6592   [(set (match_operand:VF 0 "register_operand"                "=&vr,  &vr")
6593         (if_then_else:VF
6594           (unspec:<VM>
6595             [(match_operand:<VM> 1 "vector_mask_operand"     "vmWc1,vmWc1")
6596              (match_operand 4 "vector_length_operand"        "   rK,   rK")
6597              (match_operand 5 "const_int_operand"            "    i,    i")
6598              (match_operand 6 "const_int_operand"            "    i,    i")
6599              (match_operand 7 "const_int_operand"            "    i,    i")
6600              (reg:SI VL_REGNUM)
6601              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6602           (any_float:VF
6603              (match_operand:<VNCONVERT> 3 "register_operand" "   vr,   vr"))
6604           (match_operand:VF 2 "vector_merge_operand"         "   vu,    0")))]
6605   "TARGET_VECTOR"
6606   "vfwcvt.f.x<u>.v\t%0,%3%p1"
6607   [(set_attr "type" "vfwcvtitof")
6608    (set_attr "mode" "<VNCONVERT>")])
6609
6610 (define_insn "@pred_extend<mode>"
6611   [(set (match_operand:VWEXTF 0 "register_operand"                 "=&vr,  &vr")
6612         (if_then_else:VWEXTF
6613           (unspec:<VM>
6614             [(match_operand:<VM> 1 "vector_mask_operand"          "vmWc1,vmWc1")
6615              (match_operand 4 "vector_length_operand"             "   rK,   rK")
6616              (match_operand 5 "const_int_operand"                 "    i,    i")
6617              (match_operand 6 "const_int_operand"                 "    i,    i")
6618              (match_operand 7 "const_int_operand"                 "    i,    i")
6619              (reg:SI VL_REGNUM)
6620              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6621           (float_extend:VWEXTF
6622              (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand" "   vr,   vr"))
6623           (match_operand:VWEXTF 2 "vector_merge_operand"          "   vu,    0")))]
6624   "TARGET_VECTOR"
6625   "vfwcvt.f.f.v\t%0,%3%p1"
6626   [(set_attr "type" "vfwcvtftof")
6627    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6628
6629 ;; -------------------------------------------------------------------------------
6630 ;; ---- Predicated floating-point narrow conversions
6631 ;; -------------------------------------------------------------------------------
6632 ;; Includes:
6633 ;; - 13.19 Narrowing Floating-Point/Integer Type-Convert Instructions
6634 ;; -------------------------------------------------------------------------------
6635
6636 (define_insn "@pred_narrow_fcvt_x<v_su>_f<mode>"
6637   [(set (match_operand:<VNCONVERT> 0 "register_operand"        "=vd, vd, vr, vr,  &vr,  &vr")
6638         (if_then_else:<VNCONVERT>
6639           (unspec:<VM>
6640             [(match_operand:<VM> 1 "vector_mask_operand"       " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
6641              (match_operand 4 "vector_length_operand"          " rK, rK, rK, rK,   rK,   rK")
6642              (match_operand 5 "const_int_operand"              "  i,  i,  i,  i,    i,    i")
6643              (match_operand 6 "const_int_operand"              "  i,  i,  i,  i,    i,    i")
6644              (match_operand 7 "const_int_operand"              "  i,  i,  i,  i,    i,    i")
6645              (reg:SI VL_REGNUM)
6646              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6647           (unspec:<VNCONVERT>
6648              [(match_operand:VF 3 "register_operand"           "  0,  0,  0,  0,   vr,   vr")] VFCVTS)
6649           (match_operand:<VNCONVERT> 2 "vector_merge_operand"  " vu,  0, vu,  0,   vu,    0")))]
6650   "TARGET_VECTOR"
6651   "vfncvt.x<v_su>.f.w\t%0,%3%p1"
6652   [(set_attr "type" "vfncvtftoi")
6653    (set_attr "mode" "<VNCONVERT>")])
6654
6655 (define_insn "@pred_narrow_<fix_cvt><mode>"
6656   [(set (match_operand:<VNCONVERT> 0 "register_operand"        "=vd, vd, vr, vr,  &vr,  &vr")
6657         (if_then_else:<VNCONVERT>
6658           (unspec:<VM>
6659             [(match_operand:<VM> 1 "vector_mask_operand"      " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
6660              (match_operand 4 "vector_length_operand"         " rK, rK, rK, rK,   rK,   rK")
6661              (match_operand 5 "const_int_operand"             "  i,  i,  i,  i,    i,    i")
6662              (match_operand 6 "const_int_operand"             "  i,  i,  i,  i,    i,    i")
6663              (match_operand 7 "const_int_operand"             "  i,  i,  i,  i,    i,    i")
6664              (reg:SI VL_REGNUM)
6665              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6666           (any_fix:<VNCONVERT>
6667              (match_operand:VF 3 "register_operand"           "  0,  0,  0,  0,   vr,   vr"))
6668           (match_operand:<VNCONVERT> 2 "vector_merge_operand" " vu,  0, vu,  0,   vu,    0")))]
6669   "TARGET_VECTOR"
6670   "vfncvt.rtz.x<u>.f.w\t%0,%3%p1"
6671   [(set_attr "type" "vfncvtftoi")
6672    (set_attr "mode" "<VNCONVERT>")])
6673
6674 (define_insn "@pred_narrow_<float_cvt><mode>"
6675   [(set (match_operand:<VNCONVERT> 0 "register_operand"       "=vd, vd, vr, vr,  &vr,  &vr")
6676         (if_then_else:<VNCONVERT>
6677           (unspec:<VM>
6678             [(match_operand:<VM> 1 "vector_mask_operand"      " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
6679              (match_operand 4 "vector_length_operand"         " rK, rK, rK, rK,   rK,   rK")
6680              (match_operand 5 "const_int_operand"             "  i,  i,  i,  i,    i,    i")
6681              (match_operand 6 "const_int_operand"             "  i,  i,  i,  i,    i,    i")
6682              (match_operand 7 "const_int_operand"             "  i,  i,  i,  i,    i,    i")
6683              (reg:SI VL_REGNUM)
6684              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6685           (any_float:<VNCONVERT>
6686              (match_operand:VWCONVERTI 3 "register_operand"   "  0,  0,  0,  0,   vr,   vr"))
6687           (match_operand:<VNCONVERT> 2 "vector_merge_operand" " vu,  0, vu,  0,   vu,    0")))]
6688   "TARGET_VECTOR"
6689   "vfncvt.f.x<u>.w\t%0,%3%p1"
6690   [(set_attr "type" "vfncvtitof")
6691    (set_attr "mode" "<VNCONVERT>")])
6692
6693 (define_insn "@pred_trunc<mode>"
6694   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"       "=vd, vd, vr, vr,  &vr,  &vr")
6695         (if_then_else:<V_DOUBLE_TRUNC>
6696           (unspec:<VM>
6697             [(match_operand:<VM> 1 "vector_mask_operand"           " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
6698              (match_operand 4 "vector_length_operand"              " rK, rK, rK, rK,   rK,   rK")
6699              (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i,    i,    i")
6700              (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i,    i,    i")
6701              (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i,    i,    i")
6702              (reg:SI VL_REGNUM)
6703              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6704           (float_truncate:<V_DOUBLE_TRUNC>
6705              (match_operand:VWEXTF 3 "register_operand"            "  0,  0,  0,  0,   vr,   vr"))
6706           (match_operand:<V_DOUBLE_TRUNC> 2 "vector_merge_operand" " vu,  0, vu,  0,   vu,    0")))]
6707   "TARGET_VECTOR"
6708   "vfncvt.f.f.w\t%0,%3%p1"
6709   [(set_attr "type" "vfncvtftof")
6710    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6711
6712 (define_insn "@pred_rod_trunc<mode>"
6713   [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"       "=vd, vd, vr, vr,  &vr,  &vr")
6714         (if_then_else:<V_DOUBLE_TRUNC>
6715           (unspec:<VM>
6716             [(match_operand:<VM> 1 "vector_mask_operand"           " vm, vm,Wc1,Wc1,vmWc1,vmWc1")
6717              (match_operand 4 "vector_length_operand"              " rK, rK, rK, rK,   rK,   rK")
6718              (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i,    i,    i")
6719              (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i,    i,    i")
6720              (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i,    i,    i")
6721              (reg:SI VL_REGNUM)
6722              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6723           (unspec:<V_DOUBLE_TRUNC>
6724             [(float_truncate:<V_DOUBLE_TRUNC>
6725                (match_operand:VWEXTF 3 "register_operand"          "  0,  0,  0,  0,   vr,   vr"))] UNSPEC_ROD)
6726           (match_operand:<V_DOUBLE_TRUNC> 2 "vector_merge_operand" " vu,  0, vu,  0,   vu,    0")))]
6727   "TARGET_VECTOR"
6728   "vfncvt.rod.f.f.w\t%0,%3%p1"
6729   [(set_attr "type" "vfncvtftof")
6730    (set_attr "mode" "<V_DOUBLE_TRUNC>")])
6731
6732 ;; -------------------------------------------------------------------------------
6733 ;; ---- Predicated reduction operations
6734 ;; -------------------------------------------------------------------------------
6735 ;; Includes:
6736 ;; - 14.1 Vector Single-Width Integer Reduction Instructions
6737 ;; - 14.2 Vector Widening Integer Reduction Instructions
6738 ;; - 14.3 Vector Single-Width Floating-Point Reduction Instructions
6739 ;; - 14.4 Vector Widening Floating-Point Reduction Instructions
6740 ;; -------------------------------------------------------------------------------
6741
6742 ;; For reduction operations, we should have seperate patterns for
6743 ;; TARGET_MIN_VLEN == 32 and TARGET_MIN_VLEN > 32.
6744 ;; Since reduction need LMUL = 1 scalar operand as the input operand
6745 ;; and they are different.
6746 ;; For example, The LMUL = 1 corresponding mode of VNx16QImode is VNx4QImode
6747 ;; for -march=rv*zve32* wheras VNx8QImode for -march=rv*zve64*
6748 (define_insn "@pred_reduc_<reduc><mode><vlmul1>"
6749   [(set (match_operand:<VLMUL1> 0 "register_operand"          "=vd, vd, vr, vr")
6750         (unspec:<VLMUL1>
6751           [(unspec:<VM>
6752              [(match_operand:<VM> 1 "vector_mask_operand"     " vm, vm,Wc1,Wc1")
6753               (match_operand 5 "vector_length_operand"        " rK, rK, rK, rK")
6754               (match_operand 6 "const_int_operand"            "  i,  i,  i,  i")
6755               (match_operand 7 "const_int_operand"            "  i,  i,  i,  i")
6756               (reg:SI VL_REGNUM)
6757               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6758            (any_reduc:VI
6759              (vec_duplicate:VI
6760                (vec_select:<VEL>
6761                  (match_operand:<VLMUL1> 4 "register_operand" " vr, vr, vr, vr")
6762                  (parallel [(const_int 0)])))
6763              (match_operand:VI 3 "register_operand"           " vr, vr, vr, vr"))
6764            (match_operand:<VLMUL1> 2 "vector_merge_operand"   " vu,  0, vu,  0")] UNSPEC_REDUC))]
6765   "TARGET_VECTOR && TARGET_MIN_VLEN > 32"
6766   "vred<reduc>.vs\t%0,%3,%4%p1"
6767   [(set_attr "type" "vired")
6768    (set_attr "mode" "<MODE>")])
6769
6770 (define_insn "@pred_reduc_<reduc><mode><vlmul1_zve32>"
6771   [(set (match_operand:<VLMUL1_ZVE32> 0 "register_operand"          "=vd, vd, vr, vr")
6772         (unspec:<VLMUL1_ZVE32>
6773           [(unspec:<VM>
6774              [(match_operand:<VM> 1 "vector_mask_operand"           " vm, vm,Wc1,Wc1")
6775               (match_operand 5 "vector_length_operand"              " rK, rK, rK, rK")
6776               (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i")
6777               (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i")
6778               (reg:SI VL_REGNUM)
6779               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6780            (any_reduc:VI_ZVE32
6781              (vec_duplicate:VI_ZVE32
6782                (vec_select:<VEL>
6783                  (match_operand:<VLMUL1_ZVE32> 4 "register_operand" " vr, vr, vr, vr")
6784                  (parallel [(const_int 0)])))
6785              (match_operand:VI_ZVE32 3 "register_operand"           " vr, vr, vr, vr"))
6786            (match_operand:<VLMUL1_ZVE32> 2 "vector_merge_operand"   " vu,  0, vu,  0")] UNSPEC_REDUC))]
6787   "TARGET_VECTOR && TARGET_MIN_VLEN == 32"
6788   "vred<reduc>.vs\t%0,%3,%4%p1"
6789   [(set_attr "type" "vired")
6790    (set_attr "mode" "<MODE>")])
6791
6792 (define_insn "@pred_widen_reduc_plus<v_su><mode><vwlmul1>"
6793   [(set (match_operand:<VWLMUL1> 0 "register_operand"           "=&vr,  &vr")
6794         (unspec:<VWLMUL1>
6795           [(unspec:<VM>
6796              [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6797               (match_operand 5 "vector_length_operand"         "   rK,   rK")
6798               (match_operand 6 "const_int_operand"             "    i,    i")
6799               (match_operand 7 "const_int_operand"             "    i,    i")
6800               (reg:SI VL_REGNUM)
6801               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6802            (match_operand:VWI 3 "register_operand"             "   vr,   vr")
6803            (match_operand:<VWLMUL1> 4 "register_operand"       "   vr,   vr")
6804            (match_operand:<VWLMUL1> 2 "vector_merge_operand"   "   vu,    0")] WREDUC))]
6805   "TARGET_VECTOR && TARGET_MIN_VLEN > 32"
6806   "vwredsum<v_su>.vs\t%0,%3,%4%p1"
6807   [(set_attr "type" "viwred")
6808    (set_attr "mode" "<MODE>")])
6809
6810 (define_insn "@pred_widen_reduc_plus<v_su><mode><vwlmul1_zve32>"
6811   [(set (match_operand:<VWLMUL1_ZVE32> 0 "register_operand"           "=&vr,  &vr")
6812         (unspec:<VWLMUL1_ZVE32>
6813           [(unspec:<VM>
6814              [(match_operand:<VM> 1 "vector_mask_operand"            "vmWc1,vmWc1")
6815               (match_operand 5 "vector_length_operand"               "   rK,   rK")
6816               (match_operand 6 "const_int_operand"                   "    i,    i")
6817               (match_operand 7 "const_int_operand"                   "    i,    i")
6818               (reg:SI VL_REGNUM)
6819               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6820            (match_operand:VWI_ZVE32 3 "register_operand"             "   vr,   vr")
6821            (match_operand:<VWLMUL1_ZVE32> 4 "register_operand"       "   vr,   vr")
6822            (match_operand:<VWLMUL1_ZVE32> 2 "vector_merge_operand"   "   vu,    0")] WREDUC))]
6823   "TARGET_VECTOR && TARGET_MIN_VLEN == 32"
6824   "vwredsum<v_su>.vs\t%0,%3,%4%p1"
6825   [(set_attr "type" "viwred")
6826    (set_attr "mode" "<MODE>")])
6827
6828 (define_insn "@pred_reduc_<reduc><mode><vlmul1>"
6829   [(set (match_operand:<VLMUL1> 0 "register_operand"          "=vd, vd, vr, vr")
6830         (unspec:<VLMUL1>
6831           [(unspec:<VM>
6832              [(match_operand:<VM> 1 "vector_mask_operand"      " vm, vm,Wc1,Wc1")
6833               (match_operand 5 "vector_length_operand"         " rK, rK, rK, rK")
6834               (match_operand 6 "const_int_operand"             "  i,  i,  i,  i")
6835               (match_operand 7 "const_int_operand"             "  i,  i,  i,  i")
6836               (reg:SI VL_REGNUM)
6837               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6838            (any_freduc:VF
6839              (vec_duplicate:VF
6840                (vec_select:<VEL>
6841                  (match_operand:<VLMUL1> 4 "register_operand" " vr, vr, vr, vr")
6842                  (parallel [(const_int 0)])))
6843              (match_operand:VF 3 "register_operand"           " vr, vr, vr, vr"))
6844            (match_operand:<VLMUL1> 2 "vector_merge_operand"   " vu,  0, vu,  0")] UNSPEC_REDUC))]
6845   "TARGET_VECTOR && TARGET_MIN_VLEN > 32"
6846   "vfred<reduc>.vs\t%0,%3,%4%p1"
6847   [(set_attr "type" "vfredu")
6848    (set_attr "mode" "<MODE>")])
6849
6850 (define_insn "@pred_reduc_<reduc><mode><vlmul1_zve32>"
6851   [(set (match_operand:<VLMUL1_ZVE32> 0 "register_operand"          "=vd, vd, vr, vr")
6852         (unspec:<VLMUL1_ZVE32>
6853           [(unspec:<VM>
6854              [(match_operand:<VM> 1 "vector_mask_operand"           " vm, vm,Wc1,Wc1")
6855               (match_operand 5 "vector_length_operand"              " rK, rK, rK, rK")
6856               (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i")
6857               (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i")
6858               (reg:SI VL_REGNUM)
6859               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6860            (any_freduc:VF_ZVE32
6861              (vec_duplicate:VF_ZVE32
6862                (vec_select:<VEL>
6863                  (match_operand:<VLMUL1_ZVE32> 4 "register_operand" " vr, vr, vr, vr")
6864                  (parallel [(const_int 0)])))
6865              (match_operand:VF_ZVE32 3 "register_operand"           " vr, vr, vr, vr"))
6866            (match_operand:<VLMUL1_ZVE32> 2 "vector_merge_operand"   " vu,  0, vu,  0")] UNSPEC_REDUC))]
6867   "TARGET_VECTOR && TARGET_MIN_VLEN == 32"
6868   "vfred<reduc>.vs\t%0,%3,%4%p1"
6869   [(set_attr "type" "vfredu")
6870    (set_attr "mode" "<MODE>")])
6871
6872 (define_insn "@pred_reduc_plus<order><mode><vlmul1>"
6873   [(set (match_operand:<VLMUL1> 0 "register_operand"             "=vd, vd, vr, vr")
6874         (unspec:<VLMUL1>
6875           [(unspec:<VLMUL1>
6876             [(unspec:<VM>
6877                [(match_operand:<VM> 1 "vector_mask_operand"      " vm, vm,Wc1,Wc1")
6878                 (match_operand 5 "vector_length_operand"         " rK, rK, rK, rK")
6879                 (match_operand 6 "const_int_operand"             "  i,  i,  i,  i")
6880                 (match_operand 7 "const_int_operand"             "  i,  i,  i,  i")
6881                 (reg:SI VL_REGNUM)
6882                 (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6883              (plus:VF
6884                (vec_duplicate:VF
6885                  (vec_select:<VEL>
6886                    (match_operand:<VLMUL1> 4 "register_operand" " vr, vr, vr, vr")
6887                    (parallel [(const_int 0)])))
6888                (match_operand:VF 3 "register_operand"           " vr, vr, vr, vr"))
6889              (match_operand:<VLMUL1> 2 "vector_merge_operand"   " vu,  0, vu,  0")] UNSPEC_REDUC)] ORDER))]
6890   "TARGET_VECTOR && TARGET_MIN_VLEN > 32"
6891   "vfred<order>sum.vs\t%0,%3,%4%p1"
6892   [(set_attr "type" "vfred<order>")
6893    (set_attr "mode" "<MODE>")])
6894
6895 (define_insn "@pred_reduc_plus<order><mode><vlmul1_zve32>"
6896   [(set (match_operand:<VLMUL1_ZVE32> 0 "register_operand"            "=vd, vd, vr, vr")
6897         (unspec:<VLMUL1_ZVE32>
6898           [(unspec:<VLMUL1_ZVE32>
6899             [(unspec:<VM>
6900                [(match_operand:<VM> 1 "vector_mask_operand"           " vm, vm,Wc1,Wc1")
6901                 (match_operand 5 "vector_length_operand"              " rK, rK, rK, rK")
6902                 (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i")
6903                 (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i")
6904                 (reg:SI VL_REGNUM)
6905                 (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6906              (plus:VF_ZVE32
6907                (vec_duplicate:VF_ZVE32
6908                  (vec_select:<VEL>
6909                    (match_operand:<VLMUL1_ZVE32> 4 "register_operand" " vr, vr, vr, vr")
6910                    (parallel [(const_int 0)])))
6911                (match_operand:VF_ZVE32 3 "register_operand"           " vr, vr, vr, vr"))
6912              (match_operand:<VLMUL1_ZVE32> 2 "vector_merge_operand"   " vu,  0, vu,  0")] UNSPEC_REDUC)] ORDER))]
6913   "TARGET_VECTOR && TARGET_MIN_VLEN == 32"
6914   "vfred<order>sum.vs\t%0,%3,%4%p1"
6915   [(set_attr "type" "vfred<order>")
6916    (set_attr "mode" "<MODE>")])
6917
6918 (define_insn "@pred_widen_reduc_plus<order><mode><vwlmul1>"
6919   [(set (match_operand:<VWLMUL1> 0 "register_operand"             "=&vr,  &vr")
6920         (unspec:<VWLMUL1>
6921           [(unspec:<VWLMUL1>
6922             [(unspec:<VM>
6923                [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
6924                 (match_operand 5 "vector_length_operand"         "   rK,   rK")
6925                 (match_operand 6 "const_int_operand"             "    i,    i")
6926                 (match_operand 7 "const_int_operand"             "    i,    i")
6927                 (reg:SI VL_REGNUM)
6928                 (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
6929              (match_operand:VWF 3 "register_operand"             "   vr,   vr")
6930              (match_operand:<VWLMUL1> 4 "register_operand"       "   vr,   vr")
6931              (match_operand:<VWLMUL1> 2 "vector_merge_operand"   "   vu,    0")] UNSPEC_WREDUC_SUM)] ORDER))]
6932   "TARGET_VECTOR && TARGET_MIN_VLEN > 32"
6933   "vfwred<order>sum.vs\t%0,%3,%4%p1"
6934   [(set_attr "type" "vfwred<order>")
6935    (set_attr "mode" "<MODE>")])
6936
6937 ;; -------------------------------------------------------------------------------
6938 ;; ---- Predicated permutation operations
6939 ;; -------------------------------------------------------------------------------
6940 ;; Includes:
6941 ;; - 16.1 Integer Scalar Move Instructions
6942 ;; - 16.2 Floating-Point Scalar Move Instructions
6943 ;; - 16.3 Vector Slide Instructions
6944 ;; - 16.4 Vector Register Gather Instructions
6945 ;; - 16.5 Vector Compress Instruction
6946 ;; -------------------------------------------------------------------------------
6947
6948 (define_expand "@pred_extract_first<mode>"
6949   [(set (match_operand:<VEL> 0 "register_operand")
6950         (unspec:<VEL>
6951           [(vec_select:<VEL>
6952              (match_operand:VI 1 "reg_or_mem_operand")
6953              (parallel [(const_int 0)]))
6954            (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE))]
6955   "TARGET_VECTOR"
6956 {
6957   if (MEM_P (operands[1]))
6958     {
6959       /* Combine vle.v + vmv.x.s ==> lw.  */
6960       emit_move_insn (operands[0], gen_rtx_MEM (<VEL>mode, XEXP (operands[1], 0)));
6961       DONE;
6962     }
6963 })
6964
6965 (define_insn_and_split "*pred_extract_first<mode>"
6966   [(set (match_operand:<VEL> 0 "register_operand"   "=r")
6967         (unspec:<VEL>
6968           [(vec_select:<VEL>
6969              (match_operand:VI 1 "register_operand" "vr")
6970              (parallel [(const_int 0)]))
6971            (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE))]
6972   "TARGET_VECTOR"
6973   "vmv.x.s\t%0,%1"
6974   "known_gt (GET_MODE_BITSIZE (<VEL>mode), GET_MODE_BITSIZE (Pmode))"
6975   [(const_int 0)]
6976 {
6977   /* In rv32 system, we can't use vmv.x.s directly.
6978      Instead, we should generate this following code sequence:
6979        vsrl.vx v16,v8,a0
6980        vmv.x.s a1,v16
6981        vmv.x.s a0,v8  */
6982   rtx nbits = force_reg (Pmode, gen_int_mode (GET_MODE_BITSIZE (Pmode), Pmode));
6983   rtx high_bits = gen_reg_rtx (<MODE>mode);
6984   emit_insn (gen_pred_scalar (LSHIFTRT, <MODE>mode, high_bits, CONSTM1_RTX (<VM>mode),
6985                         RVV_VUNDEF (<MODE>mode), operands[1], nbits, /* vl */ const1_rtx,
6986                         gen_int_mode (riscv_vector::TAIL_ANY, Pmode),
6987                         gen_int_mode (riscv_vector::MASK_ANY, Pmode),
6988                         gen_int_mode (riscv_vector::NONVLMAX, Pmode)));
6989   emit_insn (gen_pred_extract_first_trunc (<MODE>mode,
6990                         gen_highpart (SImode, operands[0]), high_bits));
6991   emit_insn (gen_pred_extract_first_trunc (<MODE>mode,
6992                         gen_lowpart (SImode, operands[0]), operands[1]));
6993   DONE;
6994 }
6995   [(set_attr "type" "vimovvx")
6996    (set_attr "mode" "<MODE>")])
6997
6998 (define_insn "@pred_extract_first_trunc<mode>"
6999   [(set (match_operand:SI 0 "register_operand"          "=r")
7000         (truncate:SI
7001           (unspec:<VEL>
7002             [(vec_select:<VEL>
7003                (match_operand:VI_D 1 "register_operand" "vr")
7004                (parallel [(const_int 0)]))
7005              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)))]
7006   "TARGET_VECTOR"
7007   "vmv.x.s\t%0,%1"
7008   [(set_attr "type" "vimovvx")
7009    (set_attr "mode" "<MODE>")])
7010
7011 (define_expand "@pred_extract_first<mode>"
7012   [(set (match_operand:<VEL> 0 "register_operand")
7013         (unspec:<VEL>
7014           [(vec_select:<VEL>
7015              (match_operand:VF 1 "reg_or_mem_operand")
7016              (parallel [(const_int 0)]))
7017            (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE))]
7018   "TARGET_VECTOR"
7019 {
7020   if (MEM_P (operands[1]))
7021     {
7022       /* Combine vle.v + vmv.f.s ==> flw.  */
7023       emit_move_insn (operands[0], gen_rtx_MEM (<VEL>mode, XEXP (operands[1], 0)));
7024       DONE;
7025     }
7026 })
7027
7028 (define_insn "*pred_extract_first<mode>"
7029   [(set (match_operand:<VEL> 0 "register_operand"   "=f")
7030         (unspec:<VEL>
7031           [(vec_select:<VEL>
7032              (match_operand:VF 1 "register_operand" "vr")
7033              (parallel [(const_int 0)]))
7034            (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE))]
7035   "TARGET_VECTOR"
7036   "vfmv.f.s\t%0,%1"
7037   [(set_attr "type" "vfmovvf")
7038    (set_attr "mode" "<MODE>")])
7039
7040 ;; vslide instructions
7041 (define_insn "@pred_slide<ud><mode>"
7042   [(set (match_operand:V 0 "register_operand"             "<ud_constraint>")
7043         (unspec:V
7044           [(unspec:<VM>
7045              [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
7046               (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
7047               (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
7048               (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
7049               (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
7050               (reg:SI VL_REGNUM)
7051               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7052            (match_operand:V 2 "vector_merge_operand"      " vu,  0, vu,  0")
7053            (match_operand:V 3 "register_operand"          " vr, vr, vr, vr")
7054            (match_operand 4 "pmode_reg_or_uimm5_operand"  " rK, rK, rK, rK")] VSLIDES))]
7055   "TARGET_VECTOR"
7056   "vslide<ud>.v%o4\t%0,%3,%4%p1"
7057   [(set_attr "type" "vslide<ud>")
7058    (set_attr "mode" "<MODE>")])
7059
7060 ;; vslide1 instructions
7061 (define_insn "@pred_slide<ud><mode>"
7062   [(set (match_operand:VI_QHS 0 "register_operand"        "<ud_constraint>")
7063         (unspec:VI_QHS
7064           [(unspec:<VM>
7065              [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
7066               (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
7067               (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
7068               (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
7069               (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
7070               (reg:SI VL_REGNUM)
7071               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7072            (match_operand:VI_QHS 2 "vector_merge_operand" " vu,  0, vu,  0")
7073            (match_operand:VI_QHS 3 "register_operand"     " vr, vr, vr, vr")
7074            (match_operand:<VEL> 4 "reg_or_0_operand"      " rJ, rJ, rJ, rJ")] VSLIDES1))]
7075   "TARGET_VECTOR"
7076   "vslide<ud>.vx\t%0,%3,%z4%p1"
7077   [(set_attr "type" "vislide<ud>")
7078    (set_attr "mode" "<MODE>")])
7079
7080 (define_expand "@pred_slide<ud><mode>"
7081   [(set (match_operand:VI_D 0 "register_operand")
7082         (unspec:VI_D
7083           [(unspec:<VM>
7084              [(match_operand:<VM> 1 "vector_mask_operand")
7085               (match_operand 5 "reg_or_int_operand")
7086               (match_operand 6 "const_int_operand")
7087               (match_operand 7 "const_int_operand")
7088               (match_operand 8 "const_int_operand")
7089               (reg:SI VL_REGNUM)
7090               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7091            (match_operand:VI_D 2 "vector_merge_operand")
7092            (match_operand:VI_D 3 "register_operand")
7093            (match_operand:<VEL> 4 "reg_or_int_operand")] VSLIDES1))]
7094   "TARGET_VECTOR"
7095 {
7096   if (riscv_vector::slide1_sew64_helper (<UNSPEC>, <MODE>mode,
7097                                          <VDEMOTE>mode, <VMDEMOTE>mode,
7098                                          operands))
7099     DONE;
7100 })
7101
7102 (define_insn "*pred_slide<ud><mode>"
7103   [(set (match_operand:VI_D 0 "register_operand"          "<ud_constraint>")
7104         (unspec:VI_D
7105           [(unspec:<VM>
7106              [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
7107               (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
7108               (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
7109               (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
7110               (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
7111               (reg:SI VL_REGNUM)
7112               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7113            (match_operand:VI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")
7114            (match_operand:VI_D 3 "register_operand"       " vr, vr, vr, vr")
7115            (match_operand:<VEL> 4 "reg_or_0_operand"      " rJ, rJ, rJ, rJ")] VSLIDES1))]
7116   "TARGET_VECTOR"
7117   "vslide<ud>.vx\t%0,%3,%z4%p1"
7118   [(set_attr "type" "vislide<ud>")
7119    (set_attr "mode" "<MODE>")])
7120
7121 (define_insn "*pred_slide<ud><mode>_extended"
7122   [(set (match_operand:VI_D 0 "register_operand"          "<ud_constraint>")
7123         (unspec:VI_D
7124           [(unspec:<VM>
7125              [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
7126               (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
7127               (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
7128               (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
7129               (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
7130               (reg:SI VL_REGNUM)
7131               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7132            (match_operand:VI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")
7133            (match_operand:VI_D 3 "register_operand"       " vr, vr, vr, vr")
7134            (sign_extend:<VEL>
7135              (match_operand:<VSUBEL> 4 "reg_or_0_operand" " rJ, rJ, rJ, rJ"))] VSLIDES1))]
7136   "TARGET_VECTOR"
7137   "vslide<ud>.vx\t%0,%3,%z4%p1"
7138   [(set_attr "type" "vislide<ud>")
7139    (set_attr "mode" "<MODE>")])
7140
7141 ;; vfslide1 instructions
7142 (define_insn "@pred_slide<ud><mode>"
7143   [(set (match_operand:VF 0 "register_operand"            "<ud_constraint>")
7144         (unspec:VF
7145           [(unspec:<VM>
7146              [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
7147               (match_operand 5 "vector_length_operand"    " rK, rK, rK, rK")
7148               (match_operand 6 "const_int_operand"        "  i,  i,  i,  i")
7149               (match_operand 7 "const_int_operand"        "  i,  i,  i,  i")
7150               (match_operand 8 "const_int_operand"        "  i,  i,  i,  i")
7151               (reg:SI VL_REGNUM)
7152               (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7153            (match_operand:VF 2 "vector_merge_operand"     " vu,  0, vu,  0")
7154            (match_operand:VF 3 "register_operand"         " vr, vr, vr, vr")
7155            (match_operand:<VEL> 4 "register_operand"      "  f,  f,  f,  f")] VFSLIDES1))]
7156   "TARGET_VECTOR"
7157   "vfslide<ud>.vf\t%0,%3,%4%p1"
7158   [(set_attr "type" "vfslide<ud>")
7159    (set_attr "mode" "<MODE>")])
7160
7161 ;; vrgather
7162 (define_insn "@pred_gather<mode>"
7163   [(set (match_operand:V 0 "register_operand"              "=&vr,  &vr")
7164         (if_then_else:V
7165           (unspec:<VM>
7166             [(match_operand:<VM> 1 "vector_mask_operand"  "vmWc1,vmWc1")
7167              (match_operand 5 "vector_length_operand"     "   rK,   rK")
7168              (match_operand 6 "const_int_operand"         "    i,    i")
7169              (match_operand 7 "const_int_operand"         "    i,    i")
7170              (match_operand 8 "const_int_operand"         "    i,    i")
7171              (reg:SI VL_REGNUM)
7172              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7173           (unspec:V
7174             [(match_operand:V 3 "register_operand"        "   vr,   vr")
7175              (match_operand:<VINDEX> 4 "register_operand" "   vr,   vr")] UNSPEC_VRGATHER)
7176           (match_operand:V 2 "vector_merge_operand"       "   vu,    0")))]
7177   "TARGET_VECTOR"
7178   "vrgather.vv\t%0,%3,%4%p1"
7179   [(set_attr "type" "vgather")
7180    (set_attr "mode" "<MODE>")])
7181
7182 (define_insn "@pred_gather<mode>_scalar"
7183   [(set (match_operand:V 0 "register_operand"               "=&vr,  &vr")
7184         (if_then_else:V
7185           (unspec:<VM>
7186             [(match_operand:<VM> 1 "vector_mask_operand"   "vmWc1,vmWc1")
7187              (match_operand 5 "vector_length_operand"      "   rK,   rK")
7188              (match_operand 6 "const_int_operand"          "    i,    i")
7189              (match_operand 7 "const_int_operand"          "    i,    i")
7190              (match_operand 8 "const_int_operand"          "    i,    i")
7191              (reg:SI VL_REGNUM)
7192              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7193           (unspec:V
7194             [(match_operand:V 3 "register_operand"         "   vr,   vr")
7195              (match_operand 4 "pmode_reg_or_uimm5_operand" "   rK,   rK")] UNSPEC_VRGATHER)
7196           (match_operand:V 2 "vector_merge_operand"        "   vu,    0")))]
7197   "TARGET_VECTOR"
7198   "vrgather.v%o4\t%0,%3,%4%p1"
7199   [(set_attr "type" "vgather")
7200    (set_attr "mode" "<MODE>")])
7201
7202 ;; vrgatherei16
7203 (define_insn "@pred_gatherei16<mode>"
7204   [(set (match_operand:VEI16 0 "register_operand"              "=&vr,  &vr")
7205         (if_then_else:VEI16
7206           (unspec:<VM>
7207             [(match_operand:<VM> 1 "vector_mask_operand"      "vmWc1,vmWc1")
7208              (match_operand 5 "vector_length_operand"         "   rK,   rK")
7209              (match_operand 6 "const_int_operand"             "    i,    i")
7210              (match_operand 7 "const_int_operand"             "    i,    i")
7211              (match_operand 8 "const_int_operand"             "    i,    i")
7212              (reg:SI VL_REGNUM)
7213              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7214           (unspec:VEI16
7215             [(match_operand:VEI16 3 "register_operand"        "   vr,   vr")
7216              (match_operand:<VINDEXEI16> 4 "register_operand" "   vr,   vr")] UNSPEC_VRGATHEREI16)
7217           (match_operand:VEI16 2 "vector_merge_operand"       "   vu,    0")))]
7218   "TARGET_VECTOR"
7219   "vrgatherei16.vv\t%0,%3,%4%p1"
7220   [(set_attr "type" "vgather")
7221    (set_attr "mode" "<MODE>")])
7222
7223 ;; vcompress
7224 (define_insn "@pred_compress<mode>"
7225   [(set (match_operand:V 0 "register_operand"            "=&vr,  &vr")
7226         (unspec:V
7227           [(unspec:<VM>
7228             [(match_operand:<VM> 3 "register_operand"    "  vm,  vm")
7229              (match_operand 4 "vector_length_operand"    "  rK,  rK")
7230              (match_operand 5 "const_int_operand"        "   i,   i")
7231              (match_operand 6 "const_int_operand"        "   i,   i")
7232              (reg:SI VL_REGNUM)
7233              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7234            (match_operand:V 2 "register_operand"         "  vr,  vr")
7235            (match_operand:V 1 "vector_merge_operand"     "  vu,   0")] UNSPEC_VCOMPRESS))]
7236   "TARGET_VECTOR"
7237   "vcompress.vm\t%0,%2,%3"
7238   [(set_attr "type" "vcompress")
7239    (set_attr "mode" "<MODE>")])
7240
7241 ;; -------------------------------------------------------------------------------
7242 ;; ---- Predicated Fault-Only-First loads
7243 ;; -------------------------------------------------------------------------------
7244 ;; Includes:
7245 ;; - 7.7. Unit-stride Fault-Only-First Loads
7246 ;; -------------------------------------------------------------------------------
7247
7248 (define_insn "read_vlsi"
7249   [(set (match_operand:SI 0 "register_operand" "=r")
7250         (reg:SI VL_REGNUM))]
7251   "TARGET_VECTOR"
7252   "csrr\t%0,vl"
7253   [(set_attr "type" "rdvl")
7254    (set_attr "mode" "SI")])
7255
7256 (define_insn "read_vldi_zero_extend"
7257   [(set (match_operand:DI 0 "register_operand" "=r")
7258         (zero_extend:DI (reg:SI VL_REGNUM)))]
7259   "TARGET_VECTOR && TARGET_64BIT"
7260   "csrr\t%0,vl"
7261   [(set_attr "type" "rdvl")
7262    (set_attr "mode" "DI")])
7263
7264 (define_insn "@pred_fault_load<mode>"
7265   [(set (match_operand:V 0 "register_operand"              "=vd,    vd,    vr,    vr")
7266         (if_then_else:V
7267           (unspec:<VM>
7268             [(match_operand:<VM> 1 "vector_mask_operand" "   vm,    vm,   Wc1,   Wc1")
7269              (match_operand 4 "vector_length_operand"    "   rK,    rK,    rK,    rK")
7270              (match_operand 5 "const_int_operand"        "    i,     i,     i,     i")
7271              (match_operand 6 "const_int_operand"        "    i,     i,     i,     i")
7272              (match_operand 7 "const_int_operand"        "    i,     i,     i,     i")
7273              (reg:SI VL_REGNUM)
7274              (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
7275           (unspec:V
7276             [(match_operand:V 3 "memory_operand"         "    m,     m,     m,     m")] UNSPEC_VLEFF)
7277           (match_operand:V 2 "vector_merge_operand"      "   vu,     0,    vu,     0")))
7278    (set (reg:SI VL_REGNUM) (unspec:SI [(match_dup 0)] UNSPEC_VLEFF))]
7279   "TARGET_VECTOR"
7280   "vle<sew>ff.v\t%0,%3%p1"
7281   [(set_attr "type" "vldff")
7282    (set_attr "mode" "<MODE>")])