gcc/config/rs6000/mma.md

   1 ;; Matrix-Multiply Assist (MMA) patterns.
   2 ;; Copyright (C) 2020-2024 Free Software Foundation, Inc.
   3 ;; Contributed by Peter Bergner <bergner@linux.ibm.com> and
   4 ;;                Michael Meissner <meissner@linux.ibm.com>
   5 ;;
   6 ;; This file is part of GCC.
   7 ;;
   8 ;; GCC is free software; you can redistribute it and/or modify it
   9 ;; under the terms of the GNU General Public License as published
  10 ;; by the Free Software Foundation; either version 3, or (at your
  11 ;; option) any later version.
  12 ;;
  13 ;; GCC is distributed in the hope that it will be useful, but WITHOUT
  14 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  15 ;; or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  16 ;; License for more details.
  17 ;;
  18 ;; You should have received a copy of the GNU General Public License
  19 ;; along with GCC; see the file COPYING3.  If not see
  20 ;; <http://www.gnu.org/licenses/>.
  21
  22 ;; The MMA patterns use the multi-register XOmode and OOmode opaque
  23 ;; modes to implement the target specific __vector_quad and
  24 ;; __vector_pair types that the MMA built-in functions reference.  We
  25 ;; use OPAQUE_MODE to prevent anything from trying to open them up.
  26
  27 (define_constants [(MAX_MMA_OPERANDS 7)])
  28
  29 ;; Constants for creating unspecs
  30
  31 (define_c_enum "unspec"
  32   [UNSPEC_VSX_ASSEMBLE
  33    UNSPEC_MMA_EXTRACT
  34    UNSPEC_MMA_PMXVBF16GER2
  35    UNSPEC_MMA_PMXVBF16GER2NN
  36    UNSPEC_MMA_PMXVBF16GER2NP
  37    UNSPEC_MMA_PMXVBF16GER2PN
  38    UNSPEC_MMA_PMXVBF16GER2PP
  39    UNSPEC_MMA_PMXVF16GER2
  40    UNSPEC_MMA_PMXVF16GER2NN
  41    UNSPEC_MMA_PMXVF16GER2NP
  42    UNSPEC_MMA_PMXVF16GER2PN
  43    UNSPEC_MMA_PMXVF16GER2PP
  44    UNSPEC_MMA_PMXVF32GER
  45    UNSPEC_MMA_PMXVF32GERNN
  46    UNSPEC_MMA_PMXVF32GERNP
  47    UNSPEC_MMA_PMXVF32GERPN
  48    UNSPEC_MMA_PMXVF32GERPP
  49    UNSPEC_MMA_PMXVF64GER
  50    UNSPEC_MMA_PMXVF64GERNN
  51    UNSPEC_MMA_PMXVF64GERNP
  52    UNSPEC_MMA_PMXVF64GERPN
  53    UNSPEC_MMA_PMXVF64GERPP
  54    UNSPEC_MMA_PMXVI16GER2
  55    UNSPEC_MMA_PMXVI16GER2PP
  56    UNSPEC_MMA_PMXVI16GER2S
  57    UNSPEC_MMA_PMXVI16GER2SPP
  58    UNSPEC_MMA_PMXVI4GER8
  59    UNSPEC_MMA_PMXVI4GER8PP
  60    UNSPEC_MMA_PMXVI8GER4
  61    UNSPEC_MMA_PMXVI8GER4PP
  62    UNSPEC_MMA_PMXVI8GER4SPP
  63    UNSPEC_MMA_XVBF16GER2
  64    UNSPEC_MMA_XVBF16GER2NN
  65    UNSPEC_MMA_XVBF16GER2NP
  66    UNSPEC_MMA_XVBF16GER2PN
  67    UNSPEC_MMA_XVBF16GER2PP
  68    UNSPEC_MMA_XVF16GER2
  69    UNSPEC_MMA_XVF16GER2NN
  70    UNSPEC_MMA_XVF16GER2NP
  71    UNSPEC_MMA_XVF16GER2PN
  72    UNSPEC_MMA_XVF16GER2PP
  73    UNSPEC_MMA_XVF32GER
  74    UNSPEC_MMA_XVF32GERNN
  75    UNSPEC_MMA_XVF32GERNP
  76    UNSPEC_MMA_XVF32GERPN
  77    UNSPEC_MMA_XVF32GERPP
  78    UNSPEC_MMA_XVF64GER
  79    UNSPEC_MMA_XVF64GERNN
  80    UNSPEC_MMA_XVF64GERNP
  81    UNSPEC_MMA_XVF64GERPN
  82    UNSPEC_MMA_XVF64GERPP
  83    UNSPEC_MMA_XVI16GER2
  84    UNSPEC_MMA_XVI16GER2PP
  85    UNSPEC_MMA_XVI16GER2S
  86    UNSPEC_MMA_XVI16GER2SPP
  87    UNSPEC_MMA_XVI4GER8
  88    UNSPEC_MMA_XVI4GER8PP
  89    UNSPEC_MMA_XVI8GER4
  90    UNSPEC_MMA_XVI8GER4PP
  91    UNSPEC_MMA_XVI8GER4SPP
  92    UNSPEC_MMA_XXMFACC
  93    UNSPEC_MMA_XXMTACC
  94   ])
  95
  96 (define_c_enum "unspecv"
  97   [UNSPECV_MMA_ASSEMBLE
  98    UNSPECV_MMA_XXSETACCZ
  99   ])
 100
 101 ;; MMA instructions with 1 accumulator argument
 102 (define_int_iterator MMA_ACC            [UNSPEC_MMA_XXMFACC
 103                                          UNSPEC_MMA_XXMTACC])
 104
 105 ;; MMA instructions with 2 vector arguments
 106 (define_int_iterator MMA_VV             [UNSPEC_MMA_XVI4GER8
 107                                          UNSPEC_MMA_XVI8GER4
 108                                          UNSPEC_MMA_XVI16GER2
 109                                          UNSPEC_MMA_XVI16GER2S
 110                                          UNSPEC_MMA_XVF16GER2
 111                                          UNSPEC_MMA_XVBF16GER2
 112                                          UNSPEC_MMA_XVF32GER])
 113
 114 ;; MMA instructions with 1 accumulator and 2 vector arguments
 115 (define_int_iterator MMA_AVV            [UNSPEC_MMA_XVI4GER8PP
 116                                          UNSPEC_MMA_XVI8GER4PP
 117                                          UNSPEC_MMA_XVI8GER4SPP
 118                                          UNSPEC_MMA_XVI16GER2PP
 119                                          UNSPEC_MMA_XVI16GER2SPP
 120                                          UNSPEC_MMA_XVF16GER2PP
 121                                          UNSPEC_MMA_XVF16GER2PN
 122                                          UNSPEC_MMA_XVF16GER2NP
 123                                          UNSPEC_MMA_XVF16GER2NN
 124                                          UNSPEC_MMA_XVBF16GER2PP
 125                                          UNSPEC_MMA_XVBF16GER2PN
 126                                          UNSPEC_MMA_XVBF16GER2NP
 127                                          UNSPEC_MMA_XVBF16GER2NN
 128                                          UNSPEC_MMA_XVF32GERPP
 129                                          UNSPEC_MMA_XVF32GERPN
 130                                          UNSPEC_MMA_XVF32GERNP
 131                                          UNSPEC_MMA_XVF32GERNN])
 132
 133 ;; MMA instructions with 1 vector pair and 1 vector arguments
 134 (define_int_iterator MMA_PV             [UNSPEC_MMA_XVF64GER])
 135
 136 ;; MMA instructions with 1 accumulator, 1 vector pair and 1 vector arguments
 137 (define_int_iterator MMA_APV            [UNSPEC_MMA_XVF64GERPP
 138                                          UNSPEC_MMA_XVF64GERPN
 139                                          UNSPEC_MMA_XVF64GERNP
 140                                          UNSPEC_MMA_XVF64GERNN])
 141
 142 ;; MMA instructions with 2 vector, 2 4-bit and 1 8-bit arguments
 143 (define_int_iterator MMA_VVI4I4I8       [UNSPEC_MMA_PMXVI4GER8])
 144
 145 ;; MMA instructions with 1 accumulator, 2 vector, 2 4-bit and 1 8-bit arguments
 146 (define_int_iterator MMA_AVVI4I4I8      [UNSPEC_MMA_PMXVI4GER8PP])
 147
 148 ;; MMA instructions with 2 vector, 2 4-bit and 1 2-bit arguments
 149 (define_int_iterator MMA_VVI4I4I2       [UNSPEC_MMA_PMXVI16GER2
 150                                          UNSPEC_MMA_PMXVI16GER2S
 151                                          UNSPEC_MMA_PMXVF16GER2
 152                                          UNSPEC_MMA_PMXVBF16GER2])
 153
 154 ;; MMA instructions with 1 accumulator, 2 vector, 2 4-bit and 1 2-bit arguments
 155 (define_int_iterator MMA_AVVI4I4I2      [UNSPEC_MMA_PMXVI16GER2PP
 156                                          UNSPEC_MMA_PMXVI16GER2SPP
 157                                          UNSPEC_MMA_PMXVF16GER2PP
 158                                          UNSPEC_MMA_PMXVF16GER2PN
 159                                          UNSPEC_MMA_PMXVF16GER2NP
 160                                          UNSPEC_MMA_PMXVF16GER2NN
 161                                          UNSPEC_MMA_PMXVBF16GER2PP
 162                                          UNSPEC_MMA_PMXVBF16GER2PN
 163                                          UNSPEC_MMA_PMXVBF16GER2NP
 164                                          UNSPEC_MMA_PMXVBF16GER2NN])
 165
 166 ;; MMA instructions with 2 vector and 2 4-bit arguments
 167 (define_int_iterator MMA_VVI4I4         [UNSPEC_MMA_PMXVF32GER])
 168
 169 ;; MMA instructions with 1 accumulator, 2 vector and 2 4-bit arguments
 170 (define_int_iterator MMA_AVVI4I4        [UNSPEC_MMA_PMXVF32GERPP
 171                                          UNSPEC_MMA_PMXVF32GERPN
 172                                          UNSPEC_MMA_PMXVF32GERNP
 173                                          UNSPEC_MMA_PMXVF32GERNN])
 174
 175 ;; MMA instructions with 2 vector, 1 4-bit and 1 2-bit arguments
 176 (define_int_iterator MMA_PVI4I2         [UNSPEC_MMA_PMXVF64GER])
 177
 178 ;; MMA instructions with 1 accumulator, 2 vector, 1 4-bit and 1 2-bit arguments
 179 (define_int_iterator MMA_APVI4I2        [UNSPEC_MMA_PMXVF64GERPP
 180                                          UNSPEC_MMA_PMXVF64GERPN
 181                                          UNSPEC_MMA_PMXVF64GERNP
 182                                          UNSPEC_MMA_PMXVF64GERNN])
 183
 184 ;; MMA instructions with 2 vector and 3 4-bit arguments
 185 (define_int_iterator MMA_VVI4I4I4       [UNSPEC_MMA_PMXVI8GER4])
 186
 187 ;; MMA instructions with 1 accumulator, 2 vector and 3 4-bit arguments
 188 (define_int_iterator MMA_AVVI4I4I4      [UNSPEC_MMA_PMXVI8GER4PP
 189                                          UNSPEC_MMA_PMXVI8GER4SPP])
 190
 191 (define_int_attr acc            [(UNSPEC_MMA_XXMFACC            "xxmfacc")
 192                                  (UNSPEC_MMA_XXMTACC            "xxmtacc")])
 193
 194 (define_int_attr vv             [(UNSPEC_MMA_XVI4GER8           "xvi4ger8")
 195                                  (UNSPEC_MMA_XVI8GER4           "xvi8ger4")
 196                                  (UNSPEC_MMA_XVI16GER2          "xvi16ger2")
 197                                  (UNSPEC_MMA_XVI16GER2S         "xvi16ger2s")
 198                                  (UNSPEC_MMA_XVF16GER2          "xvf16ger2")
 199                                  (UNSPEC_MMA_XVBF16GER2         "xvbf16ger2")
 200                                  (UNSPEC_MMA_XVF32GER           "xvf32ger")])
 201
 202 (define_int_attr avv            [(UNSPEC_MMA_XVI4GER8PP         "xvi4ger8pp")
 203                                  (UNSPEC_MMA_XVI8GER4PP         "xvi8ger4pp")
 204                                  (UNSPEC_MMA_XVI8GER4SPP        "xvi8ger4spp")
 205                                  (UNSPEC_MMA_XVI16GER2PP        "xvi16ger2pp")
 206                                  (UNSPEC_MMA_XVI16GER2SPP       "xvi16ger2spp")
 207                                  (UNSPEC_MMA_XVF16GER2PP        "xvf16ger2pp")
 208                                  (UNSPEC_MMA_XVF16GER2PN        "xvf16ger2pn")
 209                                  (UNSPEC_MMA_XVF16GER2NP        "xvf16ger2np")
 210                                  (UNSPEC_MMA_XVF16GER2NN        "xvf16ger2nn")
 211                                  (UNSPEC_MMA_XVBF16GER2PP       "xvbf16ger2pp")
 212                                  (UNSPEC_MMA_XVBF16GER2PN       "xvbf16ger2pn")
 213                                  (UNSPEC_MMA_XVBF16GER2NP       "xvbf16ger2np")
 214                                  (UNSPEC_MMA_XVBF16GER2NN       "xvbf16ger2nn")
 215                                  (UNSPEC_MMA_XVF32GERPP         "xvf32gerpp")
 216                                  (UNSPEC_MMA_XVF32GERPN         "xvf32gerpn")
 217                                  (UNSPEC_MMA_XVF32GERNP         "xvf32gernp")
 218                                  (UNSPEC_MMA_XVF32GERNN         "xvf32gernn")])
 219
 220 (define_int_attr pv             [(UNSPEC_MMA_XVF64GER           "xvf64ger")])
 221
 222 (define_int_attr apv            [(UNSPEC_MMA_XVF64GERPP         "xvf64gerpp")
 223                                  (UNSPEC_MMA_XVF64GERPN         "xvf64gerpn")
 224                                  (UNSPEC_MMA_XVF64GERNP         "xvf64gernp")
 225                                  (UNSPEC_MMA_XVF64GERNN         "xvf64gernn")])
 226
 227 (define_int_attr vvi4i4i8       [(UNSPEC_MMA_PMXVI4GER8         "pmxvi4ger8")])
 228
 229 (define_int_attr avvi4i4i8      [(UNSPEC_MMA_PMXVI4GER8PP       "pmxvi4ger8pp")])
 230
 231 (define_int_attr vvi4i4i2       [(UNSPEC_MMA_PMXVI16GER2        "pmxvi16ger2")
 232                                  (UNSPEC_MMA_PMXVI16GER2S       "pmxvi16ger2s")
 233                                  (UNSPEC_MMA_PMXVF16GER2        "pmxvf16ger2")
 234                                  (UNSPEC_MMA_PMXVBF16GER2       "pmxvbf16ger2")])
 235
 236 (define_int_attr avvi4i4i2      [(UNSPEC_MMA_PMXVI16GER2PP      "pmxvi16ger2pp")
 237                                  (UNSPEC_MMA_PMXVI16GER2SPP     "pmxvi16ger2spp")
 238                                  (UNSPEC_MMA_PMXVF16GER2PP      "pmxvf16ger2pp")
 239                                  (UNSPEC_MMA_PMXVF16GER2PN      "pmxvf16ger2pn")
 240                                  (UNSPEC_MMA_PMXVF16GER2NP      "pmxvf16ger2np")
 241                                  (UNSPEC_MMA_PMXVF16GER2NN      "pmxvf16ger2nn")
 242                                  (UNSPEC_MMA_PMXVBF16GER2PP     "pmxvbf16ger2pp")
 243                                  (UNSPEC_MMA_PMXVBF16GER2PN     "pmxvbf16ger2pn")
 244                                  (UNSPEC_MMA_PMXVBF16GER2NP     "pmxvbf16ger2np")
 245                                  (UNSPEC_MMA_PMXVBF16GER2NN     "pmxvbf16ger2nn")])
 246
 247 (define_int_attr vvi4i4         [(UNSPEC_MMA_PMXVF32GER         "pmxvf32ger")])
 248
 249 (define_int_attr avvi4i4        [(UNSPEC_MMA_PMXVF32GERPP       "pmxvf32gerpp")
 250                                  (UNSPEC_MMA_PMXVF32GERPN       "pmxvf32gerpn")
 251                                  (UNSPEC_MMA_PMXVF32GERNP       "pmxvf32gernp")
 252                                  (UNSPEC_MMA_PMXVF32GERNN       "pmxvf32gernn")])
 253
 254 (define_int_attr pvi4i2         [(UNSPEC_MMA_PMXVF64GER         "pmxvf64ger")])
 255
 256 (define_int_attr apvi4i2        [(UNSPEC_MMA_PMXVF64GERPP       "pmxvf64gerpp")
 257                                  (UNSPEC_MMA_PMXVF64GERPN       "pmxvf64gerpn")
 258                                  (UNSPEC_MMA_PMXVF64GERNP       "pmxvf64gernp")
 259                                  (UNSPEC_MMA_PMXVF64GERNN       "pmxvf64gernn")])
 260
 261 (define_int_attr vvi4i4i4       [(UNSPEC_MMA_PMXVI8GER4         "pmxvi8ger4")])
 262
 263 (define_int_attr avvi4i4i4      [(UNSPEC_MMA_PMXVI8GER4PP       "pmxvi8ger4pp")
 264                                  (UNSPEC_MMA_PMXVI8GER4SPP      "pmxvi8ger4spp")])
 265
 266
 267 ;; Vector pair support.  OOmode can only live in VSRs.
 268 (define_expand "movoo"
 269   [(set (match_operand:OO 0 "nonimmediate_operand")
 270         (match_operand:OO 1 "input_operand"))]
 271   ""
 272 {
 273   if (TARGET_MMA)
 274     {
 275       rs6000_emit_move (operands[0], operands[1], OOmode);
 276       DONE;
 277     }
 278   else if (currently_expanding_to_rtl && seen_error ())
 279     {
 280       /* PR103353 shows we may want to continue to expand the __builtin_vsx_lxvp
 281          built-in function, even if we have already emitted error messages about
 282          some missing required conditions.  As shown in that PR, without one
 283          explicit mov optab on OOmode provided, it would call emit_move_insn
 284          recursively.  So we allow this pattern to be generated when we are
 285          expanding to RTL and have seen errors.  It would not cause further ICEs
 286          as the compilation would stop soon after expanding.  */
 287     }
 288   else if (rs6000_opaque_type_invalid_use_p (currently_expanding_gimple_stmt))
 289     ;
 290   else
 291     /* Catch unexpected cases.  */
 292     gcc_assert (false);
 293 })
 294
 295 (define_insn_and_split "*movoo"
 296   [(set (match_operand:OO 0 "nonimmediate_operand" "=wa,ZwO,wa")
 297         (match_operand:OO 1 "input_operand" "ZwO,wa,wa"))]
 298   "TARGET_MMA
 299    && (gpc_reg_operand (operands[0], OOmode)
 300        || gpc_reg_operand (operands[1], OOmode))"
 301   "@
 302    lxvp%X1 %x0,%1
 303    stxvp%X0 %x1,%0
 304    #"
 305   "&& reload_completed
 306    && (!MEM_P (operands[0]) && !MEM_P (operands[1]))"
 307   [(const_int 0)]
 308 {
 309   rs6000_split_multireg_move (operands[0], operands[1]);
 310   DONE;
 311 }
 312   [(set_attr "type" "vecload,vecstore,veclogical")
 313    (set_attr "size" "256")
 314    (set_attr "length" "*,*,8")])
 315
 316 \f
 317 ;; Vector quad support.  XOmode can only live in FPRs.
 318 (define_expand "movxo"
 319   [(set (match_operand:XO 0 "nonimmediate_operand")
 320         (match_operand:XO 1 "input_operand"))]
 321   ""
 322 {
 323   if (TARGET_MMA)
 324     {
 325       rs6000_emit_move (operands[0], operands[1], XOmode);
 326       DONE;
 327     }
 328   else if (currently_expanding_to_rtl && seen_error ())
 329     {
 330       /* PR103353 shows we may want to continue to expand the __builtin_vsx_lxvp
 331          built-in function, even if we have already emitted error messages about
 332          some missing required conditions.  So do the same handlings for XOmode
 333          as OOmode here.  */
 334     }
 335   else if (rs6000_opaque_type_invalid_use_p (currently_expanding_gimple_stmt))
 336     ;
 337   else
 338     /* Catch unexpected cases.  */
 339     gcc_assert (false);
 340 })
 341
 342 (define_insn_and_split "*movxo"
 343   [(set (match_operand:XO 0 "nonimmediate_operand" "=d,ZwO,d")
 344         (match_operand:XO 1 "input_operand" "ZwO,d,d"))]
 345   "TARGET_MMA
 346    && (gpc_reg_operand (operands[0], XOmode)
 347        || gpc_reg_operand (operands[1], XOmode))"
 348   "@
 349    #
 350    #
 351    #"
 352   "&& reload_completed"
 353   [(const_int 0)]
 354 {
 355   rs6000_split_multireg_move (operands[0], operands[1]);
 356   DONE;
 357 }
 358   [(set_attr "type" "vecload,vecstore,veclogical")
 359    (set_attr "length" "*,*,16")
 360    (set_attr "max_prefixed_insns" "2,2,*")])
 361
 362 (define_expand "vsx_assemble_pair"
 363   [(match_operand:OO 0 "vsx_register_operand")
 364    (match_operand:V16QI 1 "mma_assemble_input_operand")
 365    (match_operand:V16QI 2 "mma_assemble_input_operand")]
 366   "TARGET_MMA"
 367 {
 368   rtx src = gen_rtx_UNSPEC (OOmode,
 369                             gen_rtvec (2, operands[1], operands[2]),
 370                             UNSPEC_VSX_ASSEMBLE);
 371   emit_move_insn (operands[0], src);
 372   DONE;
 373 })
 374
 375 ;; We cannot update the two output registers atomically, so mark the output
 376 ;; as an early clobber so we don't accidentally clobber the input operands.  */
 377
 378 (define_insn_and_split "*vsx_assemble_pair"
 379   [(set (match_operand:OO 0 "vsx_register_operand" "=&wa")
 380         (unspec:OO [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa")
 381                     (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa")]
 382                    UNSPEC_VSX_ASSEMBLE))]
 383   "TARGET_MMA"
 384   "#"
 385   "&& reload_completed"
 386   [(const_int 0)]
 387 {
 388   rtx src = gen_rtx_UNSPEC (OOmode,
 389                             gen_rtvec (2, operands[1], operands[2]),
 390                             UNSPEC_VSX_ASSEMBLE);
 391   rs6000_split_multireg_move (operands[0], src);
 392   DONE;
 393 })
 394
 395 (define_expand "vsx_disassemble_pair"
 396   [(match_operand:V16QI 0 "mma_disassemble_output_operand")
 397    (match_operand:OO 1 "vsx_register_operand")
 398    (match_operand 2 "const_0_to_1_operand")]
 399   "TARGET_MMA"
 400 {
 401   rtx src;
 402   int regoff = INTVAL (operands[2]);
 403   src = gen_rtx_UNSPEC (V16QImode,
 404                         gen_rtvec (2, operands[1], GEN_INT (regoff)),
 405                         UNSPEC_MMA_EXTRACT);
 406   emit_move_insn (operands[0], src);
 407   DONE;
 408 })
 409
 410 (define_insn_and_split "*vsx_disassemble_pair"
 411   [(set (match_operand:V16QI 0 "mma_disassemble_output_operand" "=mwa")
 412        (unspec:V16QI [(match_operand:OO 1 "vsx_register_operand" "wa")
 413                       (match_operand 2 "const_0_to_1_operand")]
 414                       UNSPEC_MMA_EXTRACT))]
 415   "TARGET_MMA
 416    && vsx_register_operand (operands[1], OOmode)"
 417   "#"
 418   "&& reload_completed"
 419   [(const_int 0)]
 420 {
 421   int reg = REGNO (operands[1]);
 422   int regoff = INTVAL (operands[2]);
 423   rtx src = gen_rtx_REG (V16QImode, reg + regoff);
 424   emit_move_insn (operands[0], src);
 425   DONE;
 426 })
 427
 428 (define_expand "mma_assemble_acc"
 429   [(match_operand:XO 0 "fpr_reg_operand")
 430    (match_operand:V16QI 1 "mma_assemble_input_operand")
 431    (match_operand:V16QI 2 "mma_assemble_input_operand")
 432    (match_operand:V16QI 3 "mma_assemble_input_operand")
 433    (match_operand:V16QI 4 "mma_assemble_input_operand")]
 434   "TARGET_MMA"
 435 {
 436   rtx src = gen_rtx_UNSPEC_VOLATILE (XOmode,
 437                                      gen_rtvec (4, operands[1], operands[2],
 438                                                 operands[3], operands[4]),
 439                                      UNSPECV_MMA_ASSEMBLE);
 440   emit_move_insn (operands[0], src);
 441   DONE;
 442 })
 443
 444 ;; We cannot update the four output registers atomically, so mark the output
 445 ;; as an early clobber so we don't accidentally clobber the input operands.  */
 446
 447 (define_insn_and_split "*mma_assemble_acc"
 448   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d")
 449         (unspec_volatile:XO
 450           [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa")
 451            (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa")
 452            (match_operand:V16QI 3 "mma_assemble_input_operand" "mwa")
 453            (match_operand:V16QI 4 "mma_assemble_input_operand" "mwa")]
 454           UNSPECV_MMA_ASSEMBLE))]
 455   "TARGET_MMA
 456    && fpr_reg_operand (operands[0], XOmode)"
 457   "#"
 458   "&& reload_completed"
 459   [(const_int 0)]
 460 {
 461   rtx src = gen_rtx_UNSPEC_VOLATILE (XOmode,
 462                                      gen_rtvec (4, operands[1], operands[2],
 463                                                 operands[3], operands[4]),
 464                                      UNSPECV_MMA_ASSEMBLE);
 465   rs6000_split_multireg_move (operands[0], src);
 466   DONE;
 467 })
 468
 469 (define_expand "mma_disassemble_acc"
 470   [(match_operand:V16QI 0 "mma_disassemble_output_operand")
 471    (match_operand:XO 1 "fpr_reg_operand")
 472    (match_operand 2 "const_0_to_3_operand")]
 473   "TARGET_MMA"
 474 {
 475   rtx src;
 476   int regoff = INTVAL (operands[2]);
 477   src = gen_rtx_UNSPEC (V16QImode,
 478                         gen_rtvec (2, operands[1], GEN_INT (regoff)),
 479                         UNSPEC_MMA_EXTRACT);
 480   emit_move_insn (operands[0], src);
 481   DONE;
 482 })
 483
 484 (define_insn_and_split "*mma_disassemble_acc"
 485   [(set (match_operand:V16QI 0 "mma_disassemble_output_operand" "=mwa")
 486        (unspec:V16QI [(match_operand:XO 1 "fpr_reg_operand" "d")
 487                       (match_operand 2 "const_0_to_3_operand")]
 488                       UNSPEC_MMA_EXTRACT))]
 489   "TARGET_MMA
 490    && fpr_reg_operand (operands[1], XOmode)"
 491   "#"
 492   "&& reload_completed"
 493   [(const_int 0)]
 494 {
 495   int reg = REGNO (operands[1]);
 496   int regoff = INTVAL (operands[2]);
 497   rtx src = gen_rtx_REG (V16QImode, reg + regoff);
 498   emit_move_insn (operands[0], src);
 499   DONE;
 500 })
 501
 502 ;; MMA instructions that do not use their accumulators as an input, still
 503 ;; must not allow their vector operands to overlap the registers used by
 504 ;; the accumulator.  We enforce this by marking the output as early clobber.
 505
 506 (define_insn "mma_<acc>"
 507   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d")
 508         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0")]
 509                     MMA_ACC))]
 510   "TARGET_MMA"
 511   "<acc> %A0"
 512   [(set_attr "type" "mma")])
 513
 514 ;; We can't have integer constants in XOmode so we wrap this in an
 515 ;; UNSPEC_VOLATILE.
 516
 517 (define_insn "mma_xxsetaccz"
 518   [(set (match_operand:XO 0 "fpr_reg_operand" "=d")
 519         (unspec_volatile:XO [(const_int 0)]
 520                             UNSPECV_MMA_XXSETACCZ))]
 521   "TARGET_MMA"
 522   "xxsetaccz %A0"
 523   [(set_attr "type" "mma")])
 524
 525 (define_insn "mma_<vv>"
 526   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 527         (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
 528                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")]
 529                     MMA_VV))]
 530   "TARGET_MMA"
 531   "<vv> %A0,%x1,%x2"
 532   [(set_attr "type" "mma")])
 533
 534 (define_insn "mma_<avv>"
 535   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 536         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
 537                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 538                     (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")]
 539                     MMA_AVV))]
 540   "TARGET_MMA"
 541   "<avv> %A0,%x2,%x3"
 542   [(set_attr "type" "mma")])
 543
 544 (define_insn "mma_<pv>"
 545   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 546         (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "v,?wa")
 547                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")]
 548                     MMA_PV))]
 549   "TARGET_MMA"
 550   "<pv> %A0,%x1,%x2"
 551   [(set_attr "type" "mma")])
 552
 553 (define_insn "mma_<apv>"
 554   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 555         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
 556                     (match_operand:OO 2 "vsx_register_operand" "v,?wa")
 557                     (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")]
 558                     MMA_APV))]
 559   "TARGET_MMA"
 560   "<apv> %A0,%x2,%x3"
 561   [(set_attr "type" "mma")])
 562
 563 (define_insn "mma_<vvi4i4i8>"
 564   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 565         (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
 566                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 567                     (match_operand:SI 3 "const_0_to_15_operand" "n,n")
 568                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 569                     (match_operand:SI 5 "u8bit_cint_operand" "n,n")]
 570                     MMA_VVI4I4I8))]
 571   "TARGET_MMA"
 572   "<vvi4i4i8> %A0,%x1,%x2,%3,%4,%5"
 573   [(set_attr "type" "mma")
 574    (set_attr "prefixed" "yes")])
 575
 576 (define_insn "mma_<avvi4i4i8>"
 577   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 578         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
 579                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 580                     (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
 581                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 582                     (match_operand:SI 5 "const_0_to_15_operand" "n,n")
 583                     (match_operand:SI 6 "u8bit_cint_operand" "n,n")]
 584                     MMA_AVVI4I4I8))]
 585   "TARGET_MMA"
 586   "<avvi4i4i8> %A0,%x2,%x3,%4,%5,%6"
 587   [(set_attr "type" "mma")
 588    (set_attr "prefixed" "yes")])
 589
 590 (define_insn "mma_<vvi4i4i2>"
 591   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 592         (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
 593                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 594                     (match_operand:SI 3 "const_0_to_15_operand" "n,n")
 595                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 596                     (match_operand:SI 5 "const_0_to_3_operand" "n,n")]
 597                     MMA_VVI4I4I2))]
 598   "TARGET_MMA"
 599   "<vvi4i4i2> %A0,%x1,%x2,%3,%4,%5"
 600   [(set_attr "type" "mma")
 601    (set_attr "prefixed" "yes")])
 602
 603 (define_insn "mma_<avvi4i4i2>"
 604   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 605         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
 606                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 607                     (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
 608                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 609                     (match_operand:SI 5 "const_0_to_15_operand" "n,n")
 610                     (match_operand:SI 6 "const_0_to_3_operand" "n,n")]
 611                     MMA_AVVI4I4I2))]
 612   "TARGET_MMA"
 613   "<avvi4i4i2> %A0,%x2,%x3,%4,%5,%6"
 614   [(set_attr "type" "mma")
 615    (set_attr "prefixed" "yes")])
 616
 617 (define_insn "mma_<vvi4i4>"
 618   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 619         (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
 620                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 621                     (match_operand:SI 3 "const_0_to_15_operand" "n,n")
 622                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")]
 623                     MMA_VVI4I4))]
 624   "TARGET_MMA"
 625   "<vvi4i4> %A0,%x1,%x2,%3,%4"
 626   [(set_attr "type" "mma")
 627    (set_attr "prefixed" "yes")])
 628
 629 (define_insn "mma_<avvi4i4>"
 630   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 631         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
 632                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 633                     (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
 634                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 635                     (match_operand:SI 5 "const_0_to_15_operand" "n,n")]
 636                     MMA_AVVI4I4))]
 637   "TARGET_MMA"
 638   "<avvi4i4> %A0,%x2,%x3,%4,%5"
 639   [(set_attr "type" "mma")
 640    (set_attr "prefixed" "yes")])
 641
 642 (define_insn "mma_<pvi4i2>"
 643   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 644         (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "v,?wa")
 645                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 646                     (match_operand:SI 3 "const_0_to_15_operand" "n,n")
 647                     (match_operand:SI 4 "const_0_to_3_operand" "n,n")]
 648                     MMA_PVI4I2))]
 649   "TARGET_MMA"
 650   "<pvi4i2> %A0,%x1,%x2,%3,%4"
 651   [(set_attr "type" "mma")
 652    (set_attr "prefixed" "yes")])
 653
 654 (define_insn "mma_<apvi4i2>"
 655   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 656         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
 657                     (match_operand:OO 2 "vsx_register_operand" "v,?wa")
 658                     (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
 659                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 660                     (match_operand:SI 5 "const_0_to_3_operand" "n,n")]
 661                     MMA_APVI4I2))]
 662   "TARGET_MMA"
 663   "<apvi4i2> %A0,%x2,%x3,%4,%5"
 664   [(set_attr "type" "mma")
 665    (set_attr "prefixed" "yes")])
 666
 667 (define_insn "mma_<vvi4i4i4>"
 668   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 669         (unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
 670                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 671                     (match_operand:SI 3 "const_0_to_15_operand" "n,n")
 672                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 673                     (match_operand:SI 5 "const_0_to_15_operand" "n,n")]
 674                     MMA_VVI4I4I4))]
 675   "TARGET_MMA"
 676   "<vvi4i4i4> %A0,%x1,%x2,%3,%4,%5"
 677   [(set_attr "type" "mma")
 678    (set_attr "prefixed" "yes")])
 679
 680 (define_insn "mma_<avvi4i4i4>"
 681   [(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
 682         (unspec:XO [(match_operand:XO 1 "fpr_reg_operand" "0,0")
 683                     (match_operand:V16QI 2 "vsx_register_operand" "v,?wa")
 684                     (match_operand:V16QI 3 "vsx_register_operand" "v,?wa")
 685                     (match_operand:SI 4 "const_0_to_15_operand" "n,n")
 686                     (match_operand:SI 5 "const_0_to_15_operand" "n,n")
 687                     (match_operand:SI 6 "const_0_to_15_operand" "n,n")]
 688                     MMA_AVVI4I4I4))]
 689   "TARGET_MMA"
 690   "<avvi4i4i4> %A0,%x2,%x3,%4,%5,%6"
 691   [(set_attr "type" "mma")
 692    (set_attr "prefixed" "yes")])