hw/intc/riscv_aplic.c

   1 /*
   2  * RISC-V APLIC (Advanced Platform Level Interrupt Controller)
   3  *
   4  * Copyright (c) 2021 Western Digital Corporation or its affiliates.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2 or later, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include "qemu/osdep.h"
  20 #include "qapi/error.h"
  21 #include "qemu/log.h"
  22 #include "qemu/module.h"
  23 #include "qemu/error-report.h"
  24 #include "qemu/bswap.h"
  25 #include "exec/address-spaces.h"
  26 #include "hw/sysbus.h"
  27 #include "hw/pci/msi.h"
  28 #include "hw/boards.h"
  29 #include "hw/qdev-properties.h"
  30 #include "hw/intc/riscv_aplic.h"
  31 #include "hw/irq.h"
  32 #include "target/riscv/cpu.h"
  33 #include "sysemu/sysemu.h"
  34 #include "sysemu/kvm.h"
  35 #include "kvm/kvm_riscv.h"
  36 #include "migration/vmstate.h"
  37
  38 #define APLIC_MAX_IDC                  (1UL << 14)
  39 #define APLIC_MAX_SOURCE               1024
  40 #define APLIC_MIN_IPRIO_BITS           1
  41 #define APLIC_MAX_IPRIO_BITS           8
  42 #define APLIC_MAX_CHILDREN             1024
  43
  44 #define APLIC_DOMAINCFG                0x0000
  45 #define APLIC_DOMAINCFG_RDONLY         0x80000000
  46 #define APLIC_DOMAINCFG_IE             (1 << 8)
  47 #define APLIC_DOMAINCFG_DM             (1 << 2)
  48 #define APLIC_DOMAINCFG_BE             (1 << 0)
  49
  50 #define APLIC_SOURCECFG_BASE           0x0004
  51 #define APLIC_SOURCECFG_D              (1 << 10)
  52 #define APLIC_SOURCECFG_CHILDIDX_MASK  0x000003ff
  53 #define APLIC_SOURCECFG_SM_MASK        0x00000007
  54 #define APLIC_SOURCECFG_SM_INACTIVE    0x0
  55 #define APLIC_SOURCECFG_SM_DETACH      0x1
  56 #define APLIC_SOURCECFG_SM_EDGE_RISE   0x4
  57 #define APLIC_SOURCECFG_SM_EDGE_FALL   0x5
  58 #define APLIC_SOURCECFG_SM_LEVEL_HIGH  0x6
  59 #define APLIC_SOURCECFG_SM_LEVEL_LOW   0x7
  60
  61 #define APLIC_MMSICFGADDR              0x1bc0
  62 #define APLIC_MMSICFGADDRH             0x1bc4
  63 #define APLIC_SMSICFGADDR              0x1bc8
  64 #define APLIC_SMSICFGADDRH             0x1bcc
  65
  66 #define APLIC_xMSICFGADDRH_L           (1UL << 31)
  67 #define APLIC_xMSICFGADDRH_HHXS_MASK   0x1f
  68 #define APLIC_xMSICFGADDRH_HHXS_SHIFT  24
  69 #define APLIC_xMSICFGADDRH_LHXS_MASK   0x7
  70 #define APLIC_xMSICFGADDRH_LHXS_SHIFT  20
  71 #define APLIC_xMSICFGADDRH_HHXW_MASK   0x7
  72 #define APLIC_xMSICFGADDRH_HHXW_SHIFT  16
  73 #define APLIC_xMSICFGADDRH_LHXW_MASK   0xf
  74 #define APLIC_xMSICFGADDRH_LHXW_SHIFT  12
  75 #define APLIC_xMSICFGADDRH_BAPPN_MASK  0xfff
  76
  77 #define APLIC_xMSICFGADDR_PPN_SHIFT    12
  78
  79 #define APLIC_xMSICFGADDR_PPN_HART(__lhxs) \
  80     ((1UL << (__lhxs)) - 1)
  81
  82 #define APLIC_xMSICFGADDR_PPN_LHX_MASK(__lhxw) \
  83     ((1UL << (__lhxw)) - 1)
  84 #define APLIC_xMSICFGADDR_PPN_LHX_SHIFT(__lhxs) \
  85     ((__lhxs))
  86 #define APLIC_xMSICFGADDR_PPN_LHX(__lhxw, __lhxs) \
  87     (APLIC_xMSICFGADDR_PPN_LHX_MASK(__lhxw) << \
  88      APLIC_xMSICFGADDR_PPN_LHX_SHIFT(__lhxs))
  89
  90 #define APLIC_xMSICFGADDR_PPN_HHX_MASK(__hhxw) \
  91     ((1UL << (__hhxw)) - 1)
  92 #define APLIC_xMSICFGADDR_PPN_HHX_SHIFT(__hhxs) \
  93     ((__hhxs) + APLIC_xMSICFGADDR_PPN_SHIFT)
  94 #define APLIC_xMSICFGADDR_PPN_HHX(__hhxw, __hhxs) \
  95     (APLIC_xMSICFGADDR_PPN_HHX_MASK(__hhxw) << \
  96      APLIC_xMSICFGADDR_PPN_HHX_SHIFT(__hhxs))
  97
  98 #define APLIC_xMSICFGADDRH_VALID_MASK   \
  99     (APLIC_xMSICFGADDRH_L | \
 100      (APLIC_xMSICFGADDRH_HHXS_MASK << APLIC_xMSICFGADDRH_HHXS_SHIFT) | \
 101      (APLIC_xMSICFGADDRH_LHXS_MASK << APLIC_xMSICFGADDRH_LHXS_SHIFT) | \
 102      (APLIC_xMSICFGADDRH_HHXW_MASK << APLIC_xMSICFGADDRH_HHXW_SHIFT) | \
 103      (APLIC_xMSICFGADDRH_LHXW_MASK << APLIC_xMSICFGADDRH_LHXW_SHIFT) | \
 104      APLIC_xMSICFGADDRH_BAPPN_MASK)
 105
 106 #define APLIC_SETIP_BASE               0x1c00
 107 #define APLIC_SETIPNUM                 0x1cdc
 108
 109 #define APLIC_CLRIP_BASE               0x1d00
 110 #define APLIC_CLRIPNUM                 0x1ddc
 111
 112 #define APLIC_SETIE_BASE               0x1e00
 113 #define APLIC_SETIENUM                 0x1edc
 114
 115 #define APLIC_CLRIE_BASE               0x1f00
 116 #define APLIC_CLRIENUM                 0x1fdc
 117
 118 #define APLIC_SETIPNUM_LE              0x2000
 119 #define APLIC_SETIPNUM_BE              0x2004
 120
 121 #define APLIC_ISTATE_PENDING           (1U << 0)
 122 #define APLIC_ISTATE_ENABLED           (1U << 1)
 123 #define APLIC_ISTATE_ENPEND            (APLIC_ISTATE_ENABLED | \
 124                                         APLIC_ISTATE_PENDING)
 125 #define APLIC_ISTATE_INPUT             (1U << 8)
 126
 127 #define APLIC_GENMSI                   0x3000
 128
 129 #define APLIC_TARGET_BASE              0x3004
 130 #define APLIC_TARGET_HART_IDX_SHIFT    18
 131 #define APLIC_TARGET_HART_IDX_MASK     0x3fff
 132 #define APLIC_TARGET_GUEST_IDX_SHIFT   12
 133 #define APLIC_TARGET_GUEST_IDX_MASK    0x3f
 134 #define APLIC_TARGET_IPRIO_MASK        0xff
 135 #define APLIC_TARGET_EIID_MASK         0x7ff
 136
 137 #define APLIC_IDC_BASE                 0x4000
 138 #define APLIC_IDC_SIZE                 32
 139
 140 #define APLIC_IDC_IDELIVERY            0x00
 141
 142 #define APLIC_IDC_IFORCE               0x04
 143
 144 #define APLIC_IDC_ITHRESHOLD           0x08
 145
 146 #define APLIC_IDC_TOPI                 0x18
 147 #define APLIC_IDC_TOPI_ID_SHIFT        16
 148 #define APLIC_IDC_TOPI_ID_MASK         0x3ff
 149 #define APLIC_IDC_TOPI_PRIO_MASK       0xff
 150
 151 #define APLIC_IDC_CLAIMI               0x1c
 152
 153 /*
 154  * KVM AIA only supports APLIC MSI, fallback to QEMU emulation if we want to use
 155  * APLIC Wired.
 156  */
 157 static bool is_kvm_aia(bool msimode)
 158 {
 159     return kvm_irqchip_in_kernel() && msimode;
 160 }
 161
 162 static uint32_t riscv_aplic_read_input_word(RISCVAPLICState *aplic,
 163                                             uint32_t word)
 164 {
 165     uint32_t i, irq, sourcecfg, sm, raw_input, irq_inverted, ret = 0;
 166
 167     for (i = 0; i < 32; i++) {
 168         irq = word * 32 + i;
 169         if (!irq || aplic->num_irqs <= irq) {
 170             continue;
 171         }
 172
 173         sourcecfg = aplic->sourcecfg[irq];
 174         if (sourcecfg & APLIC_SOURCECFG_D) {
 175             continue;
 176         }
 177
 178         sm = sourcecfg & APLIC_SOURCECFG_SM_MASK;
 179         if (sm == APLIC_SOURCECFG_SM_INACTIVE) {
 180             continue;
 181         }
 182
 183         raw_input = (aplic->state[irq] & APLIC_ISTATE_INPUT) ? 1 : 0;
 184         irq_inverted = (sm == APLIC_SOURCECFG_SM_LEVEL_LOW ||
 185                         sm == APLIC_SOURCECFG_SM_EDGE_FALL) ? 1 : 0;
 186         ret |= (raw_input ^ irq_inverted) << i;
 187     }
 188
 189     return ret;
 190 }
 191
 192 static uint32_t riscv_aplic_read_pending_word(RISCVAPLICState *aplic,
 193                                               uint32_t word)
 194 {
 195     uint32_t i, irq, ret = 0;
 196
 197     for (i = 0; i < 32; i++) {
 198         irq = word * 32 + i;
 199         if (!irq || aplic->num_irqs <= irq) {
 200             continue;
 201         }
 202
 203         ret |= ((aplic->state[irq] & APLIC_ISTATE_PENDING) ? 1 : 0) << i;
 204     }
 205
 206     return ret;
 207 }
 208
 209 static void riscv_aplic_set_pending_raw(RISCVAPLICState *aplic,
 210                                         uint32_t irq, bool pending)
 211 {
 212     if (pending) {
 213         aplic->state[irq] |= APLIC_ISTATE_PENDING;
 214     } else {
 215         aplic->state[irq] &= ~APLIC_ISTATE_PENDING;
 216     }
 217 }
 218
 219 static void riscv_aplic_set_pending(RISCVAPLICState *aplic,
 220                                     uint32_t irq, bool pending)
 221 {
 222     uint32_t sourcecfg, sm;
 223
 224     if ((irq <= 0) || (aplic->num_irqs <= irq)) {
 225         return;
 226     }
 227
 228     sourcecfg = aplic->sourcecfg[irq];
 229     if (sourcecfg & APLIC_SOURCECFG_D) {
 230         return;
 231     }
 232
 233     sm = sourcecfg & APLIC_SOURCECFG_SM_MASK;
 234     if (sm == APLIC_SOURCECFG_SM_INACTIVE) {
 235         return;
 236     }
 237
 238     if ((sm == APLIC_SOURCECFG_SM_LEVEL_HIGH) ||
 239         (sm == APLIC_SOURCECFG_SM_LEVEL_LOW)) {
 240         if (!aplic->msimode || (aplic->msimode && !pending)) {
 241             return;
 242         }
 243         if ((aplic->state[irq] & APLIC_ISTATE_INPUT) &&
 244             (sm == APLIC_SOURCECFG_SM_LEVEL_LOW)) {
 245             return;
 246         }
 247         if (!(aplic->state[irq] & APLIC_ISTATE_INPUT) &&
 248             (sm == APLIC_SOURCECFG_SM_LEVEL_HIGH)) {
 249             return;
 250         }
 251     }
 252
 253     riscv_aplic_set_pending_raw(aplic, irq, pending);
 254 }
 255
 256 static void riscv_aplic_set_pending_word(RISCVAPLICState *aplic,
 257                                          uint32_t word, uint32_t value,
 258                                          bool pending)
 259 {
 260     uint32_t i, irq;
 261
 262     for (i = 0; i < 32; i++) {
 263         irq = word * 32 + i;
 264         if (!irq || aplic->num_irqs <= irq) {
 265             continue;
 266         }
 267
 268         if (value & (1U << i)) {
 269             riscv_aplic_set_pending(aplic, irq, pending);
 270         }
 271     }
 272 }
 273
 274 static uint32_t riscv_aplic_read_enabled_word(RISCVAPLICState *aplic,
 275                                               int word)
 276 {
 277     uint32_t i, irq, ret = 0;
 278
 279     for (i = 0; i < 32; i++) {
 280         irq = word * 32 + i;
 281         if (!irq || aplic->num_irqs <= irq) {
 282             continue;
 283         }
 284
 285         ret |= ((aplic->state[irq] & APLIC_ISTATE_ENABLED) ? 1 : 0) << i;
 286     }
 287
 288     return ret;
 289 }
 290
 291 static void riscv_aplic_set_enabled_raw(RISCVAPLICState *aplic,
 292                                         uint32_t irq, bool enabled)
 293 {
 294     if (enabled) {
 295         aplic->state[irq] |= APLIC_ISTATE_ENABLED;
 296     } else {
 297         aplic->state[irq] &= ~APLIC_ISTATE_ENABLED;
 298     }
 299 }
 300
 301 static void riscv_aplic_set_enabled(RISCVAPLICState *aplic,
 302                                     uint32_t irq, bool enabled)
 303 {
 304     uint32_t sourcecfg, sm;
 305
 306     if ((irq <= 0) || (aplic->num_irqs <= irq)) {
 307         return;
 308     }
 309
 310     sourcecfg = aplic->sourcecfg[irq];
 311     if (sourcecfg & APLIC_SOURCECFG_D) {
 312         return;
 313     }
 314
 315     sm = sourcecfg & APLIC_SOURCECFG_SM_MASK;
 316     if (sm == APLIC_SOURCECFG_SM_INACTIVE) {
 317         return;
 318     }
 319
 320     riscv_aplic_set_enabled_raw(aplic, irq, enabled);
 321 }
 322
 323 static void riscv_aplic_set_enabled_word(RISCVAPLICState *aplic,
 324                                          uint32_t word, uint32_t value,
 325                                          bool enabled)
 326 {
 327     uint32_t i, irq;
 328
 329     for (i = 0; i < 32; i++) {
 330         irq = word * 32 + i;
 331         if (!irq || aplic->num_irqs <= irq) {
 332             continue;
 333         }
 334
 335         if (value & (1U << i)) {
 336             riscv_aplic_set_enabled(aplic, irq, enabled);
 337         }
 338     }
 339 }
 340
 341 static void riscv_aplic_msi_send(RISCVAPLICState *aplic,
 342                                  uint32_t hart_idx, uint32_t guest_idx,
 343                                  uint32_t eiid)
 344 {
 345     uint64_t addr;
 346     MemTxResult result;
 347     RISCVAPLICState *aplic_m;
 348     uint32_t lhxs, lhxw, hhxs, hhxw, group_idx, msicfgaddr, msicfgaddrH;
 349
 350     aplic_m = aplic;
 351     while (aplic_m && !aplic_m->mmode) {
 352         aplic_m = aplic_m->parent;
 353     }
 354     if (!aplic_m) {
 355         qemu_log_mask(LOG_GUEST_ERROR, "%s: m-level APLIC not found\n",
 356                       __func__);
 357         return;
 358     }
 359
 360     if (aplic->mmode) {
 361         msicfgaddr = aplic_m->mmsicfgaddr;
 362         msicfgaddrH = aplic_m->mmsicfgaddrH;
 363     } else {
 364         msicfgaddr = aplic_m->smsicfgaddr;
 365         msicfgaddrH = aplic_m->smsicfgaddrH;
 366     }
 367
 368     lhxs = (msicfgaddrH >> APLIC_xMSICFGADDRH_LHXS_SHIFT) &
 369             APLIC_xMSICFGADDRH_LHXS_MASK;
 370     lhxw = (msicfgaddrH >> APLIC_xMSICFGADDRH_LHXW_SHIFT) &
 371             APLIC_xMSICFGADDRH_LHXW_MASK;
 372     hhxs = (msicfgaddrH >> APLIC_xMSICFGADDRH_HHXS_SHIFT) &
 373             APLIC_xMSICFGADDRH_HHXS_MASK;
 374     hhxw = (msicfgaddrH >> APLIC_xMSICFGADDRH_HHXW_SHIFT) &
 375             APLIC_xMSICFGADDRH_HHXW_MASK;
 376
 377     group_idx = hart_idx >> lhxw;
 378     hart_idx &= APLIC_xMSICFGADDR_PPN_LHX_MASK(lhxw);
 379
 380     addr = msicfgaddr;
 381     addr |= ((uint64_t)(msicfgaddrH & APLIC_xMSICFGADDRH_BAPPN_MASK)) << 32;
 382     addr |= ((uint64_t)(group_idx & APLIC_xMSICFGADDR_PPN_HHX_MASK(hhxw))) <<
 383              APLIC_xMSICFGADDR_PPN_HHX_SHIFT(hhxs);
 384     addr |= ((uint64_t)(hart_idx & APLIC_xMSICFGADDR_PPN_LHX_MASK(lhxw))) <<
 385              APLIC_xMSICFGADDR_PPN_LHX_SHIFT(lhxs);
 386     addr |= (uint64_t)(guest_idx & APLIC_xMSICFGADDR_PPN_HART(lhxs));
 387     addr <<= APLIC_xMSICFGADDR_PPN_SHIFT;
 388
 389     address_space_stl_le(&address_space_memory, addr,
 390                          eiid, MEMTXATTRS_UNSPECIFIED, &result);
 391     if (result != MEMTX_OK) {
 392         qemu_log_mask(LOG_GUEST_ERROR, "%s: MSI write failed for "
 393                       "hart_index=%d guest_index=%d eiid=%d\n",
 394                       __func__, hart_idx, guest_idx, eiid);
 395     }
 396 }
 397
 398 static void riscv_aplic_msi_irq_update(RISCVAPLICState *aplic, uint32_t irq)
 399 {
 400     uint32_t hart_idx, guest_idx, eiid;
 401
 402     if (!aplic->msimode || (aplic->num_irqs <= irq) ||
 403         !(aplic->domaincfg & APLIC_DOMAINCFG_IE)) {
 404         return;
 405     }
 406
 407     if ((aplic->state[irq] & APLIC_ISTATE_ENPEND) != APLIC_ISTATE_ENPEND) {
 408         return;
 409     }
 410
 411     riscv_aplic_set_pending_raw(aplic, irq, false);
 412
 413     hart_idx = aplic->target[irq] >> APLIC_TARGET_HART_IDX_SHIFT;
 414     hart_idx &= APLIC_TARGET_HART_IDX_MASK;
 415     if (aplic->mmode) {
 416         /* M-level APLIC ignores guest_index */
 417         guest_idx = 0;
 418     } else {
 419         guest_idx = aplic->target[irq] >> APLIC_TARGET_GUEST_IDX_SHIFT;
 420         guest_idx &= APLIC_TARGET_GUEST_IDX_MASK;
 421     }
 422     eiid = aplic->target[irq] & APLIC_TARGET_EIID_MASK;
 423     riscv_aplic_msi_send(aplic, hart_idx, guest_idx, eiid);
 424 }
 425
 426 static uint32_t riscv_aplic_idc_topi(RISCVAPLICState *aplic, uint32_t idc)
 427 {
 428     uint32_t best_irq, best_iprio;
 429     uint32_t irq, iprio, ihartidx, ithres;
 430
 431     if (aplic->num_harts <= idc) {
 432         return 0;
 433     }
 434
 435     ithres = aplic->ithreshold[idc];
 436     best_irq = best_iprio = UINT32_MAX;
 437     for (irq = 1; irq < aplic->num_irqs; irq++) {
 438         if ((aplic->state[irq] & APLIC_ISTATE_ENPEND) !=
 439             APLIC_ISTATE_ENPEND) {
 440             continue;
 441         }
 442
 443         ihartidx = aplic->target[irq] >> APLIC_TARGET_HART_IDX_SHIFT;
 444         ihartidx &= APLIC_TARGET_HART_IDX_MASK;
 445         if (ihartidx != idc) {
 446             continue;
 447         }
 448
 449         iprio = aplic->target[irq] & aplic->iprio_mask;
 450         if (ithres && iprio >= ithres) {
 451             continue;
 452         }
 453
 454         if (iprio < best_iprio) {
 455             best_irq = irq;
 456             best_iprio = iprio;
 457         }
 458     }
 459
 460     if (best_irq < aplic->num_irqs && best_iprio <= aplic->iprio_mask) {
 461         return (best_irq << APLIC_IDC_TOPI_ID_SHIFT) | best_iprio;
 462     }
 463
 464     return 0;
 465 }
 466
 467 static void riscv_aplic_idc_update(RISCVAPLICState *aplic, uint32_t idc)
 468 {
 469     uint32_t topi;
 470
 471     if (aplic->msimode || aplic->num_harts <= idc) {
 472         return;
 473     }
 474
 475     topi = riscv_aplic_idc_topi(aplic, idc);
 476     if ((aplic->domaincfg & APLIC_DOMAINCFG_IE) &&
 477         aplic->idelivery[idc] &&
 478         (aplic->iforce[idc] || topi)) {
 479         qemu_irq_raise(aplic->external_irqs[idc]);
 480     } else {
 481         qemu_irq_lower(aplic->external_irqs[idc]);
 482     }
 483 }
 484
 485 static uint32_t riscv_aplic_idc_claimi(RISCVAPLICState *aplic, uint32_t idc)
 486 {
 487     uint32_t irq, state, sm, topi = riscv_aplic_idc_topi(aplic, idc);
 488
 489     if (!topi) {
 490         aplic->iforce[idc] = 0;
 491         riscv_aplic_idc_update(aplic, idc);
 492         return 0;
 493     }
 494
 495     irq = (topi >> APLIC_IDC_TOPI_ID_SHIFT) & APLIC_IDC_TOPI_ID_MASK;
 496     sm = aplic->sourcecfg[irq] & APLIC_SOURCECFG_SM_MASK;
 497     state = aplic->state[irq];
 498     riscv_aplic_set_pending_raw(aplic, irq, false);
 499     if ((sm == APLIC_SOURCECFG_SM_LEVEL_HIGH) &&
 500         (state & APLIC_ISTATE_INPUT)) {
 501         riscv_aplic_set_pending_raw(aplic, irq, true);
 502     } else if ((sm == APLIC_SOURCECFG_SM_LEVEL_LOW) &&
 503                !(state & APLIC_ISTATE_INPUT)) {
 504         riscv_aplic_set_pending_raw(aplic, irq, true);
 505     }
 506     riscv_aplic_idc_update(aplic, idc);
 507
 508     return topi;
 509 }
 510
 511 static void riscv_aplic_request(void *opaque, int irq, int level)
 512 {
 513     bool update = false;
 514     RISCVAPLICState *aplic = opaque;
 515     uint32_t sourcecfg, childidx, state, idc;
 516
 517     assert((0 < irq) && (irq < aplic->num_irqs));
 518
 519     sourcecfg = aplic->sourcecfg[irq];
 520     if (sourcecfg & APLIC_SOURCECFG_D) {
 521         childidx = sourcecfg & APLIC_SOURCECFG_CHILDIDX_MASK;
 522         if (childidx < aplic->num_children) {
 523             riscv_aplic_request(aplic->children[childidx], irq, level);
 524         }
 525         return;
 526     }
 527
 528     state = aplic->state[irq];
 529     switch (sourcecfg & APLIC_SOURCECFG_SM_MASK) {
 530     case APLIC_SOURCECFG_SM_EDGE_RISE:
 531         if ((level > 0) && !(state & APLIC_ISTATE_INPUT) &&
 532             !(state & APLIC_ISTATE_PENDING)) {
 533             riscv_aplic_set_pending_raw(aplic, irq, true);
 534             update = true;
 535         }
 536         break;
 537     case APLIC_SOURCECFG_SM_EDGE_FALL:
 538         if ((level <= 0) && (state & APLIC_ISTATE_INPUT) &&
 539             !(state & APLIC_ISTATE_PENDING)) {
 540             riscv_aplic_set_pending_raw(aplic, irq, true);
 541             update = true;
 542         }
 543         break;
 544     case APLIC_SOURCECFG_SM_LEVEL_HIGH:
 545         if ((level > 0) && !(state & APLIC_ISTATE_PENDING)) {
 546             riscv_aplic_set_pending_raw(aplic, irq, true);
 547             update = true;
 548         }
 549         break;
 550     case APLIC_SOURCECFG_SM_LEVEL_LOW:
 551         if ((level <= 0) && !(state & APLIC_ISTATE_PENDING)) {
 552             riscv_aplic_set_pending_raw(aplic, irq, true);
 553             update = true;
 554         }
 555         break;
 556     default:
 557         break;
 558     }
 559
 560     if (level <= 0) {
 561         aplic->state[irq] &= ~APLIC_ISTATE_INPUT;
 562     } else {
 563         aplic->state[irq] |= APLIC_ISTATE_INPUT;
 564     }
 565
 566     if (update) {
 567         if (aplic->msimode) {
 568             riscv_aplic_msi_irq_update(aplic, irq);
 569         } else {
 570             idc = aplic->target[irq] >> APLIC_TARGET_HART_IDX_SHIFT;
 571             idc &= APLIC_TARGET_HART_IDX_MASK;
 572             riscv_aplic_idc_update(aplic, idc);
 573         }
 574     }
 575 }
 576
 577 static uint64_t riscv_aplic_read(void *opaque, hwaddr addr, unsigned size)
 578 {
 579     uint32_t irq, word, idc;
 580     RISCVAPLICState *aplic = opaque;
 581
 582     /* Reads must be 4 byte words */
 583     if ((addr & 0x3) != 0) {
 584         goto err;
 585     }
 586
 587     if (addr == APLIC_DOMAINCFG) {
 588         return APLIC_DOMAINCFG_RDONLY | aplic->domaincfg |
 589                (aplic->msimode ? APLIC_DOMAINCFG_DM : 0);
 590     } else if ((APLIC_SOURCECFG_BASE <= addr) &&
 591             (addr < (APLIC_SOURCECFG_BASE + (aplic->num_irqs - 1) * 4))) {
 592         irq  = ((addr - APLIC_SOURCECFG_BASE) >> 2) + 1;
 593         return aplic->sourcecfg[irq];
 594     } else if (aplic->mmode && aplic->msimode &&
 595                (addr == APLIC_MMSICFGADDR)) {
 596         return aplic->mmsicfgaddr;
 597     } else if (aplic->mmode && aplic->msimode &&
 598                (addr == APLIC_MMSICFGADDRH)) {
 599         return aplic->mmsicfgaddrH;
 600     } else if (aplic->mmode && aplic->msimode &&
 601                (addr == APLIC_SMSICFGADDR)) {
 602         /*
 603          * Registers SMSICFGADDR and SMSICFGADDRH are implemented only if:
 604          * (a) the interrupt domain is at machine level
 605          * (b) the domain's harts implement supervisor mode
 606          * (c) the domain has one or more child supervisor-level domains
 607          *     that support MSI delivery mode (domaincfg.DM is not read-
 608          *     only zero in at least one of the supervisor-level child
 609          * domains).
 610          */
 611         return (aplic->num_children) ? aplic->smsicfgaddr : 0;
 612     } else if (aplic->mmode && aplic->msimode &&
 613                (addr == APLIC_SMSICFGADDRH)) {
 614         return (aplic->num_children) ? aplic->smsicfgaddrH : 0;
 615     } else if ((APLIC_SETIP_BASE <= addr) &&
 616             (addr < (APLIC_SETIP_BASE + aplic->bitfield_words * 4))) {
 617         word = (addr - APLIC_SETIP_BASE) >> 2;
 618         return riscv_aplic_read_pending_word(aplic, word);
 619     } else if (addr == APLIC_SETIPNUM) {
 620         return 0;
 621     } else if ((APLIC_CLRIP_BASE <= addr) &&
 622             (addr < (APLIC_CLRIP_BASE + aplic->bitfield_words * 4))) {
 623         word = (addr - APLIC_CLRIP_BASE) >> 2;
 624         return riscv_aplic_read_input_word(aplic, word);
 625     } else if (addr == APLIC_CLRIPNUM) {
 626         return 0;
 627     } else if ((APLIC_SETIE_BASE <= addr) &&
 628             (addr < (APLIC_SETIE_BASE + aplic->bitfield_words * 4))) {
 629         word = (addr - APLIC_SETIE_BASE) >> 2;
 630         return riscv_aplic_read_enabled_word(aplic, word);
 631     } else if (addr == APLIC_SETIENUM) {
 632         return 0;
 633     } else if ((APLIC_CLRIE_BASE <= addr) &&
 634             (addr < (APLIC_CLRIE_BASE + aplic->bitfield_words * 4))) {
 635         return 0;
 636     } else if (addr == APLIC_CLRIENUM) {
 637         return 0;
 638     } else if (addr == APLIC_SETIPNUM_LE) {
 639         return 0;
 640     } else if (addr == APLIC_SETIPNUM_BE) {
 641         return 0;
 642     } else if (addr == APLIC_GENMSI) {
 643         return (aplic->msimode) ? aplic->genmsi : 0;
 644     } else if ((APLIC_TARGET_BASE <= addr) &&
 645             (addr < (APLIC_TARGET_BASE + (aplic->num_irqs - 1) * 4))) {
 646         irq = ((addr - APLIC_TARGET_BASE) >> 2) + 1;
 647         return aplic->target[irq];
 648     } else if (!aplic->msimode && (APLIC_IDC_BASE <= addr) &&
 649             (addr < (APLIC_IDC_BASE + aplic->num_harts * APLIC_IDC_SIZE))) {
 650         idc = (addr - APLIC_IDC_BASE) / APLIC_IDC_SIZE;
 651         switch (addr - (APLIC_IDC_BASE + idc * APLIC_IDC_SIZE)) {
 652         case APLIC_IDC_IDELIVERY:
 653             return aplic->idelivery[idc];
 654         case APLIC_IDC_IFORCE:
 655             return aplic->iforce[idc];
 656         case APLIC_IDC_ITHRESHOLD:
 657             return aplic->ithreshold[idc];
 658         case APLIC_IDC_TOPI:
 659             return riscv_aplic_idc_topi(aplic, idc);
 660         case APLIC_IDC_CLAIMI:
 661             return riscv_aplic_idc_claimi(aplic, idc);
 662         default:
 663             goto err;
 664         };
 665     }
 666
 667 err:
 668     qemu_log_mask(LOG_GUEST_ERROR,
 669                   "%s: Invalid register read 0x%" HWADDR_PRIx "\n",
 670                   __func__, addr);
 671     return 0;
 672 }
 673
 674 static void riscv_aplic_write(void *opaque, hwaddr addr, uint64_t value,
 675         unsigned size)
 676 {
 677     RISCVAPLICState *aplic = opaque;
 678     uint32_t irq, word, idc = UINT32_MAX;
 679
 680     /* Writes must be 4 byte words */
 681     if ((addr & 0x3) != 0) {
 682         goto err;
 683     }
 684
 685     if (addr == APLIC_DOMAINCFG) {
 686         /* Only IE bit writable at the moment */
 687         value &= APLIC_DOMAINCFG_IE;
 688         aplic->domaincfg = value;
 689     } else if ((APLIC_SOURCECFG_BASE <= addr) &&
 690             (addr < (APLIC_SOURCECFG_BASE + (aplic->num_irqs - 1) * 4))) {
 691         irq  = ((addr - APLIC_SOURCECFG_BASE) >> 2) + 1;
 692         if (!aplic->num_children && (value & APLIC_SOURCECFG_D)) {
 693             value = 0;
 694         }
 695         if (value & APLIC_SOURCECFG_D) {
 696             value &= (APLIC_SOURCECFG_D | APLIC_SOURCECFG_CHILDIDX_MASK);
 697         } else {
 698             value &= (APLIC_SOURCECFG_D | APLIC_SOURCECFG_SM_MASK);
 699         }
 700         aplic->sourcecfg[irq] = value;
 701         if ((aplic->sourcecfg[irq] & APLIC_SOURCECFG_D) ||
 702             (aplic->sourcecfg[irq] == 0)) {
 703             riscv_aplic_set_pending_raw(aplic, irq, false);
 704             riscv_aplic_set_enabled_raw(aplic, irq, false);
 705         }
 706     } else if (aplic->mmode && aplic->msimode &&
 707                (addr == APLIC_MMSICFGADDR)) {
 708         if (!(aplic->mmsicfgaddrH & APLIC_xMSICFGADDRH_L)) {
 709             aplic->mmsicfgaddr = value;
 710         }
 711     } else if (aplic->mmode && aplic->msimode &&
 712                (addr == APLIC_MMSICFGADDRH)) {
 713         if (!(aplic->mmsicfgaddrH & APLIC_xMSICFGADDRH_L)) {
 714             aplic->mmsicfgaddrH = value & APLIC_xMSICFGADDRH_VALID_MASK;
 715         }
 716     } else if (aplic->mmode && aplic->msimode &&
 717                (addr == APLIC_SMSICFGADDR)) {
 718         /*
 719          * Registers SMSICFGADDR and SMSICFGADDRH are implemented only if:
 720          * (a) the interrupt domain is at machine level
 721          * (b) the domain's harts implement supervisor mode
 722          * (c) the domain has one or more child supervisor-level domains
 723          *     that support MSI delivery mode (domaincfg.DM is not read-
 724          *     only zero in at least one of the supervisor-level child
 725          * domains).
 726          */
 727         if (aplic->num_children &&
 728             !(aplic->mmsicfgaddrH & APLIC_xMSICFGADDRH_L)) {
 729             aplic->smsicfgaddr = value;
 730         }
 731     } else if (aplic->mmode && aplic->msimode &&
 732                (addr == APLIC_SMSICFGADDRH)) {
 733         if (aplic->num_children &&
 734             !(aplic->mmsicfgaddrH & APLIC_xMSICFGADDRH_L)) {
 735             aplic->smsicfgaddrH = value & APLIC_xMSICFGADDRH_VALID_MASK;
 736         }
 737     } else if ((APLIC_SETIP_BASE <= addr) &&
 738             (addr < (APLIC_SETIP_BASE + aplic->bitfield_words * 4))) {
 739         word = (addr - APLIC_SETIP_BASE) >> 2;
 740         riscv_aplic_set_pending_word(aplic, word, value, true);
 741     } else if (addr == APLIC_SETIPNUM) {
 742         riscv_aplic_set_pending(aplic, value, true);
 743     } else if ((APLIC_CLRIP_BASE <= addr) &&
 744             (addr < (APLIC_CLRIP_BASE + aplic->bitfield_words * 4))) {
 745         word = (addr - APLIC_CLRIP_BASE) >> 2;
 746         riscv_aplic_set_pending_word(aplic, word, value, false);
 747     } else if (addr == APLIC_CLRIPNUM) {
 748         riscv_aplic_set_pending(aplic, value, false);
 749     } else if ((APLIC_SETIE_BASE <= addr) &&
 750             (addr < (APLIC_SETIE_BASE + aplic->bitfield_words * 4))) {
 751         word = (addr - APLIC_SETIE_BASE) >> 2;
 752         riscv_aplic_set_enabled_word(aplic, word, value, true);
 753     } else if (addr == APLIC_SETIENUM) {
 754         riscv_aplic_set_enabled(aplic, value, true);
 755     } else if ((APLIC_CLRIE_BASE <= addr) &&
 756             (addr < (APLIC_CLRIE_BASE + aplic->bitfield_words * 4))) {
 757         word = (addr - APLIC_CLRIE_BASE) >> 2;
 758         riscv_aplic_set_enabled_word(aplic, word, value, false);
 759     } else if (addr == APLIC_CLRIENUM) {
 760         riscv_aplic_set_enabled(aplic, value, false);
 761     } else if (addr == APLIC_SETIPNUM_LE) {
 762         riscv_aplic_set_pending(aplic, value, true);
 763     } else if (addr == APLIC_SETIPNUM_BE) {
 764         riscv_aplic_set_pending(aplic, bswap32(value), true);
 765     } else if (addr == APLIC_GENMSI) {
 766         if (aplic->msimode) {
 767             aplic->genmsi = value & ~(APLIC_TARGET_GUEST_IDX_MASK <<
 768                                       APLIC_TARGET_GUEST_IDX_SHIFT);
 769             riscv_aplic_msi_send(aplic,
 770                                  value >> APLIC_TARGET_HART_IDX_SHIFT,
 771                                  0,
 772                                  value & APLIC_TARGET_EIID_MASK);
 773         }
 774     } else if ((APLIC_TARGET_BASE <= addr) &&
 775             (addr < (APLIC_TARGET_BASE + (aplic->num_irqs - 1) * 4))) {
 776         irq = ((addr - APLIC_TARGET_BASE) >> 2) + 1;
 777         if (aplic->msimode) {
 778             aplic->target[irq] = value;
 779         } else {
 780             aplic->target[irq] = (value & ~APLIC_TARGET_IPRIO_MASK) |
 781                                  ((value & aplic->iprio_mask) ?
 782                                   (value & aplic->iprio_mask) : 1);
 783         }
 784     } else if (!aplic->msimode && (APLIC_IDC_BASE <= addr) &&
 785             (addr < (APLIC_IDC_BASE + aplic->num_harts * APLIC_IDC_SIZE))) {
 786         idc = (addr - APLIC_IDC_BASE) / APLIC_IDC_SIZE;
 787         switch (addr - (APLIC_IDC_BASE + idc * APLIC_IDC_SIZE)) {
 788         case APLIC_IDC_IDELIVERY:
 789             aplic->idelivery[idc] = value & 0x1;
 790             break;
 791         case APLIC_IDC_IFORCE:
 792             aplic->iforce[idc] = value & 0x1;
 793             break;
 794         case APLIC_IDC_ITHRESHOLD:
 795             aplic->ithreshold[idc] = value & aplic->iprio_mask;
 796             break;
 797         default:
 798             goto err;
 799         };
 800     } else {
 801         goto err;
 802     }
 803
 804     if (aplic->msimode) {
 805         for (irq = 1; irq < aplic->num_irqs; irq++) {
 806             riscv_aplic_msi_irq_update(aplic, irq);
 807         }
 808     } else {
 809         if (idc == UINT32_MAX) {
 810             for (idc = 0; idc < aplic->num_harts; idc++) {
 811                 riscv_aplic_idc_update(aplic, idc);
 812             }
 813         } else {
 814             riscv_aplic_idc_update(aplic, idc);
 815         }
 816     }
 817
 818     return;
 819
 820 err:
 821     qemu_log_mask(LOG_GUEST_ERROR,
 822                   "%s: Invalid register write 0x%" HWADDR_PRIx "\n",
 823                   __func__, addr);
 824 }
 825
 826 static const MemoryRegionOps riscv_aplic_ops = {
 827     .read = riscv_aplic_read,
 828     .write = riscv_aplic_write,
 829     .endianness = DEVICE_LITTLE_ENDIAN,
 830     .valid = {
 831         .min_access_size = 4,
 832         .max_access_size = 4
 833     }
 834 };
 835
 836 static void riscv_aplic_realize(DeviceState *dev, Error **errp)
 837 {
 838     uint32_t i;
 839     RISCVAPLICState *aplic = RISCV_APLIC(dev);
 840
 841     if (!is_kvm_aia(aplic->msimode)) {
 842         aplic->bitfield_words = (aplic->num_irqs + 31) >> 5;
 843         aplic->sourcecfg = g_new0(uint32_t, aplic->num_irqs);
 844         aplic->state = g_new0(uint32_t, aplic->num_irqs);
 845         aplic->target = g_new0(uint32_t, aplic->num_irqs);
 846         if (!aplic->msimode) {
 847             for (i = 0; i < aplic->num_irqs; i++) {
 848                 aplic->target[i] = 1;
 849             }
 850         }
 851         aplic->idelivery = g_new0(uint32_t, aplic->num_harts);
 852         aplic->iforce = g_new0(uint32_t, aplic->num_harts);
 853         aplic->ithreshold = g_new0(uint32_t, aplic->num_harts);
 854
 855         memory_region_init_io(&aplic->mmio, OBJECT(dev), &riscv_aplic_ops,
 856                               aplic, TYPE_RISCV_APLIC, aplic->aperture_size);
 857         sysbus_init_mmio(SYS_BUS_DEVICE(dev), &aplic->mmio);
 858     }
 859
 860     /*
 861      * Only root APLICs have hardware IRQ lines. All non-root APLICs
 862      * have IRQ lines delegated by their parent APLIC.
 863      */
 864     if (!aplic->parent) {
 865         if (kvm_enabled() && is_kvm_aia(aplic->msimode)) {
 866             qdev_init_gpio_in(dev, riscv_kvm_aplic_request, aplic->num_irqs);
 867         } else {
 868             qdev_init_gpio_in(dev, riscv_aplic_request, aplic->num_irqs);
 869         }
 870     }
 871
 872     /* Create output IRQ lines for non-MSI mode */
 873     if (!aplic->msimode) {
 874         aplic->external_irqs = g_malloc(sizeof(qemu_irq) * aplic->num_harts);
 875         qdev_init_gpio_out(dev, aplic->external_irqs, aplic->num_harts);
 876
 877         /* Claim the CPU interrupt to be triggered by this APLIC */
 878         for (i = 0; i < aplic->num_harts; i++) {
 879             RISCVCPU *cpu = RISCV_CPU(cpu_by_arch_id(aplic->hartid_base + i));
 880             if (riscv_cpu_claim_interrupts(cpu,
 881                 (aplic->mmode) ? MIP_MEIP : MIP_SEIP) < 0) {
 882                 error_report("%s already claimed",
 883                              (aplic->mmode) ? "MEIP" : "SEIP");
 884                 exit(1);
 885             }
 886         }
 887     }
 888
 889     msi_nonbroken = true;
 890 }
 891
 892 static Property riscv_aplic_properties[] = {
 893     DEFINE_PROP_UINT32("aperture-size", RISCVAPLICState, aperture_size, 0),
 894     DEFINE_PROP_UINT32("hartid-base", RISCVAPLICState, hartid_base, 0),
 895     DEFINE_PROP_UINT32("num-harts", RISCVAPLICState, num_harts, 0),
 896     DEFINE_PROP_UINT32("iprio-mask", RISCVAPLICState, iprio_mask, 0),
 897     DEFINE_PROP_UINT32("num-irqs", RISCVAPLICState, num_irqs, 0),
 898     DEFINE_PROP_BOOL("msimode", RISCVAPLICState, msimode, 0),
 899     DEFINE_PROP_BOOL("mmode", RISCVAPLICState, mmode, 0),
 900     DEFINE_PROP_END_OF_LIST(),
 901 };
 902
 903 static const VMStateDescription vmstate_riscv_aplic = {
 904     .name = "riscv_aplic",
 905     .version_id = 1,
 906     .minimum_version_id = 1,
 907     .fields = (const VMStateField[]) {
 908             VMSTATE_UINT32(domaincfg, RISCVAPLICState),
 909             VMSTATE_UINT32(mmsicfgaddr, RISCVAPLICState),
 910             VMSTATE_UINT32(mmsicfgaddrH, RISCVAPLICState),
 911             VMSTATE_UINT32(smsicfgaddr, RISCVAPLICState),
 912             VMSTATE_UINT32(smsicfgaddrH, RISCVAPLICState),
 913             VMSTATE_UINT32(genmsi, RISCVAPLICState),
 914             VMSTATE_VARRAY_UINT32(sourcecfg, RISCVAPLICState,
 915                                   num_irqs, 0,
 916                                   vmstate_info_uint32, uint32_t),
 917             VMSTATE_VARRAY_UINT32(state, RISCVAPLICState,
 918                                   num_irqs, 0,
 919                                   vmstate_info_uint32, uint32_t),
 920             VMSTATE_VARRAY_UINT32(target, RISCVAPLICState,
 921                                   num_irqs, 0,
 922                                   vmstate_info_uint32, uint32_t),
 923             VMSTATE_VARRAY_UINT32(idelivery, RISCVAPLICState,
 924                                   num_harts, 0,
 925                                   vmstate_info_uint32, uint32_t),
 926             VMSTATE_VARRAY_UINT32(iforce, RISCVAPLICState,
 927                                   num_harts, 0,
 928                                   vmstate_info_uint32, uint32_t),
 929             VMSTATE_VARRAY_UINT32(ithreshold, RISCVAPLICState,
 930                                   num_harts, 0,
 931                                   vmstate_info_uint32, uint32_t),
 932             VMSTATE_END_OF_LIST()
 933         }
 934 };
 935
 936 static void riscv_aplic_class_init(ObjectClass *klass, void *data)
 937 {
 938     DeviceClass *dc = DEVICE_CLASS(klass);
 939
 940     device_class_set_props(dc, riscv_aplic_properties);
 941     dc->realize = riscv_aplic_realize;
 942     dc->vmsd = &vmstate_riscv_aplic;
 943 }
 944
 945 static const TypeInfo riscv_aplic_info = {
 946     .name          = TYPE_RISCV_APLIC,
 947     .parent        = TYPE_SYS_BUS_DEVICE,
 948     .instance_size = sizeof(RISCVAPLICState),
 949     .class_init    = riscv_aplic_class_init,
 950 };
 951
 952 static void riscv_aplic_register_types(void)
 953 {
 954     type_register_static(&riscv_aplic_info);
 955 }
 956
 957 type_init(riscv_aplic_register_types)
 958
 959 /*
 960  * Add a APLIC device to another APLIC device as child for
 961  * interrupt delegation.
 962  */
 963 void riscv_aplic_add_child(DeviceState *parent, DeviceState *child)
 964 {
 965     RISCVAPLICState *caplic, *paplic;
 966
 967     assert(parent && child);
 968     caplic = RISCV_APLIC(child);
 969     paplic = RISCV_APLIC(parent);
 970
 971     assert(paplic->num_irqs == caplic->num_irqs);
 972     assert(paplic->num_children <= QEMU_APLIC_MAX_CHILDREN);
 973
 974     caplic->parent = paplic;
 975     paplic->children[paplic->num_children] = caplic;
 976     paplic->num_children++;
 977 }
 978
 979 /*
 980  * Create APLIC device.
 981  */
 982 DeviceState *riscv_aplic_create(hwaddr addr, hwaddr size,
 983     uint32_t hartid_base, uint32_t num_harts, uint32_t num_sources,
 984     uint32_t iprio_bits, bool msimode, bool mmode, DeviceState *parent)
 985 {
 986     DeviceState *dev = qdev_new(TYPE_RISCV_APLIC);
 987     uint32_t i;
 988
 989     assert(num_harts < APLIC_MAX_IDC);
 990     assert((APLIC_IDC_BASE + (num_harts * APLIC_IDC_SIZE)) <= size);
 991     assert(num_sources < APLIC_MAX_SOURCE);
 992     assert(APLIC_MIN_IPRIO_BITS <= iprio_bits);
 993     assert(iprio_bits <= APLIC_MAX_IPRIO_BITS);
 994
 995     qdev_prop_set_uint32(dev, "aperture-size", size);
 996     qdev_prop_set_uint32(dev, "hartid-base", hartid_base);
 997     qdev_prop_set_uint32(dev, "num-harts", num_harts);
 998     qdev_prop_set_uint32(dev, "iprio-mask", ((1U << iprio_bits) - 1));
 999     qdev_prop_set_uint32(dev, "num-irqs", num_sources + 1);
1000     qdev_prop_set_bit(dev, "msimode", msimode);
1001     qdev_prop_set_bit(dev, "mmode", mmode);
1002
1003     sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
1004
1005     if (!is_kvm_aia(msimode)) {
1006         sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
1007     }
1008
1009     if (parent) {
1010         riscv_aplic_add_child(parent, dev);
1011     }
1012
1013     if (!msimode) {
1014         for (i = 0; i < num_harts; i++) {
1015             CPUState *cpu = cpu_by_arch_id(hartid_base + i);
1016
1017             qdev_connect_gpio_out_named(dev, NULL, i,
1018                                         qdev_get_gpio_in(DEVICE(cpu),
1019                                             (mmode) ? IRQ_M_EXT : IRQ_S_EXT));
1020         }
1021     }
1022
1023     return dev;
1024 }