1 #include "qemu/osdep.h"
2 #include "qemu/units.h"
3 #include "qemu/error-report.h"
4 #include "hw/mem/memory-device.h"
5 #include "hw/mem/pc-dimm.h"
6 #include "hw/pci/pci.h"
7 #include "hw/qdev-properties.h"
8 #include "qapi/error.h"
10 #include "qemu/module.h"
11 #include "qemu/pmem.h"
12 #include "qemu/range.h"
14 #include "sysemu/hostmem.h"
15 #include "sysemu/numa.h"
16 #include "hw/cxl/cxl.h"
17 #include "hw/pci/msix.h"
21 /* Default CDAT entries for a memory region */
32 static int ct3_build_cdat_entries_for_mr(CDATSubHeader
**cdat_table
,
33 int dsmad_handle
, MemoryRegion
*mr
)
35 g_autofree CDATDsmas
*dsmas
= NULL
;
36 g_autofree CDATDslbis
*dslbis0
= NULL
;
37 g_autofree CDATDslbis
*dslbis1
= NULL
;
38 g_autofree CDATDslbis
*dslbis2
= NULL
;
39 g_autofree CDATDslbis
*dslbis3
= NULL
;
40 g_autofree CDATDsemts
*dsemts
= NULL
;
42 dsmas
= g_malloc(sizeof(*dsmas
));
46 *dsmas
= (CDATDsmas
) {
48 .type
= CDAT_TYPE_DSMAS
,
49 .length
= sizeof(*dsmas
),
51 .DSMADhandle
= dsmad_handle
,
52 .flags
= CDAT_DSMAS_FLAG_NV
,
54 .DPA_length
= int128_get64(mr
->size
),
57 /* For now, no memory side cache, plausiblish numbers */
58 dslbis0
= g_malloc(sizeof(*dslbis0
));
62 *dslbis0
= (CDATDslbis
) {
64 .type
= CDAT_TYPE_DSLBIS
,
65 .length
= sizeof(*dslbis0
),
67 .handle
= dsmad_handle
,
68 .flags
= HMAT_LB_MEM_MEMORY
,
69 .data_type
= HMAT_LB_DATA_READ_LATENCY
,
70 .entry_base_unit
= 10000, /* 10ns base */
71 .entry
[0] = 15, /* 150ns */
74 dslbis1
= g_malloc(sizeof(*dslbis1
));
78 *dslbis1
= (CDATDslbis
) {
80 .type
= CDAT_TYPE_DSLBIS
,
81 .length
= sizeof(*dslbis1
),
83 .handle
= dsmad_handle
,
84 .flags
= HMAT_LB_MEM_MEMORY
,
85 .data_type
= HMAT_LB_DATA_WRITE_LATENCY
,
86 .entry_base_unit
= 10000,
87 .entry
[0] = 25, /* 250ns */
90 dslbis2
= g_malloc(sizeof(*dslbis2
));
94 *dslbis2
= (CDATDslbis
) {
96 .type
= CDAT_TYPE_DSLBIS
,
97 .length
= sizeof(*dslbis2
),
99 .handle
= dsmad_handle
,
100 .flags
= HMAT_LB_MEM_MEMORY
,
101 .data_type
= HMAT_LB_DATA_READ_BANDWIDTH
,
102 .entry_base_unit
= 1000, /* GB/s */
106 dslbis3
= g_malloc(sizeof(*dslbis3
));
110 *dslbis3
= (CDATDslbis
) {
112 .type
= CDAT_TYPE_DSLBIS
,
113 .length
= sizeof(*dslbis3
),
115 .handle
= dsmad_handle
,
116 .flags
= HMAT_LB_MEM_MEMORY
,
117 .data_type
= HMAT_LB_DATA_WRITE_BANDWIDTH
,
118 .entry_base_unit
= 1000, /* GB/s */
122 dsemts
= g_malloc(sizeof(*dsemts
));
126 *dsemts
= (CDATDsemts
) {
128 .type
= CDAT_TYPE_DSEMTS
,
129 .length
= sizeof(*dsemts
),
131 .DSMAS_handle
= dsmad_handle
,
132 /* Reserved - the non volatile from DSMAS matters */
133 .EFI_memory_type_attr
= 2,
135 .DPA_length
= int128_get64(mr
->size
),
138 /* Header always at start of structure */
139 cdat_table
[CT3_CDAT_DSMAS
] = g_steal_pointer(&dsmas
);
140 cdat_table
[CT3_CDAT_DSLBIS0
] = g_steal_pointer(&dslbis0
);
141 cdat_table
[CT3_CDAT_DSLBIS1
] = g_steal_pointer(&dslbis1
);
142 cdat_table
[CT3_CDAT_DSLBIS2
] = g_steal_pointer(&dslbis2
);
143 cdat_table
[CT3_CDAT_DSLBIS3
] = g_steal_pointer(&dslbis3
);
144 cdat_table
[CT3_CDAT_DSEMTS
] = g_steal_pointer(&dsemts
);
149 static int ct3_build_cdat_table(CDATSubHeader
***cdat_table
, void *priv
)
151 g_autofree CDATSubHeader
**table
= NULL
;
152 MemoryRegion
*nonvolatile_mr
;
153 CXLType3Dev
*ct3d
= priv
;
154 int dsmad_handle
= 0;
157 if (!ct3d
->hostmem
) {
161 nonvolatile_mr
= host_memory_backend_get_memory(ct3d
->hostmem
);
162 if (!nonvolatile_mr
) {
166 table
= g_malloc0(CT3_CDAT_NUM_ENTRIES
* sizeof(*table
));
171 rc
= ct3_build_cdat_entries_for_mr(table
, dsmad_handle
++, nonvolatile_mr
);
176 *cdat_table
= g_steal_pointer(&table
);
178 return CT3_CDAT_NUM_ENTRIES
;
181 static void ct3_free_cdat_table(CDATSubHeader
**cdat_table
, int num
, void *priv
)
185 for (i
= 0; i
< num
; i
++) {
186 g_free(cdat_table
[i
]);
191 static bool cxl_doe_cdat_rsp(DOECap
*doe_cap
)
193 CDATObject
*cdat
= &CXL_TYPE3(doe_cap
->pdev
)->cxl_cstate
.cdat
;
197 CDATReq
*req
= pcie_doe_get_write_mbox_ptr(doe_cap
);
200 assert(cdat
->entry_len
);
202 /* Discard if request length mismatched */
203 if (pcie_doe_get_obj_len(req
) <
204 DIV_ROUND_UP(sizeof(CDATReq
), DWORD_BYTE
)) {
208 ent
= req
->entry_handle
;
209 base
= cdat
->entry
[ent
].base
;
210 len
= cdat
->entry
[ent
].length
;
214 .vendor_id
= CXL_VENDOR_ID
,
215 .data_obj_type
= CXL_DOE_TABLE_ACCESS
,
217 .length
= DIV_ROUND_UP((sizeof(rsp
) + len
), DWORD_BYTE
),
219 .rsp_code
= CXL_DOE_TAB_RSP
,
220 .table_type
= CXL_DOE_TAB_TYPE_CDAT
,
221 .entry_handle
= (ent
< cdat
->entry_len
- 1) ?
222 ent
+ 1 : CXL_DOE_TAB_ENT_MAX
,
225 memcpy(doe_cap
->read_mbox
, &rsp
, sizeof(rsp
));
226 memcpy(doe_cap
->read_mbox
+ DIV_ROUND_UP(sizeof(rsp
), DWORD_BYTE
),
229 doe_cap
->read_mbox_len
+= rsp
.header
.length
;
234 static uint32_t ct3d_config_read(PCIDevice
*pci_dev
, uint32_t addr
, int size
)
236 CXLType3Dev
*ct3d
= CXL_TYPE3(pci_dev
);
239 if (pcie_doe_read_config(&ct3d
->doe_cdat
, addr
, size
, &val
)) {
243 return pci_default_read_config(pci_dev
, addr
, size
);
246 static void ct3d_config_write(PCIDevice
*pci_dev
, uint32_t addr
, uint32_t val
,
249 CXLType3Dev
*ct3d
= CXL_TYPE3(pci_dev
);
251 pcie_doe_write_config(&ct3d
->doe_cdat
, addr
, val
, size
);
252 pci_default_write_config(pci_dev
, addr
, val
, size
);
256 * Null value of all Fs suggested by IEEE RA guidelines for use of
259 #define UI64_NULL ~(0ULL)
261 static void build_dvsecs(CXLType3Dev
*ct3d
)
263 CXLComponentState
*cxl_cstate
= &ct3d
->cxl_cstate
;
266 dvsec
= (uint8_t *)&(CXLDVSECDevice
){
270 .range1_size_hi
= ct3d
->hostmem
->size
>> 32,
271 .range1_size_lo
= (2 << 5) | (2 << 2) | 0x3 |
272 (ct3d
->hostmem
->size
& 0xF0000000),
276 cxl_component_create_dvsec(cxl_cstate
, CXL2_TYPE3_DEVICE
,
277 PCIE_CXL_DEVICE_DVSEC_LENGTH
,
278 PCIE_CXL_DEVICE_DVSEC
,
279 PCIE_CXL2_DEVICE_DVSEC_REVID
, dvsec
);
281 dvsec
= (uint8_t *)&(CXLDVSECRegisterLocator
){
283 .reg0_base_lo
= RBI_COMPONENT_REG
| CXL_COMPONENT_REG_BAR_IDX
,
285 .reg1_base_lo
= RBI_CXL_DEVICE_REG
| CXL_DEVICE_REG_BAR_IDX
,
288 cxl_component_create_dvsec(cxl_cstate
, CXL2_TYPE3_DEVICE
,
289 REG_LOC_DVSEC_LENGTH
, REG_LOC_DVSEC
,
290 REG_LOC_DVSEC_REVID
, dvsec
);
291 dvsec
= (uint8_t *)&(CXLDVSECDeviceGPF
){
292 .phase2_duration
= 0x603, /* 3 seconds */
293 .phase2_power
= 0x33, /* 0x33 miliwatts */
295 cxl_component_create_dvsec(cxl_cstate
, CXL2_TYPE3_DEVICE
,
296 GPF_DEVICE_DVSEC_LENGTH
, GPF_DEVICE_DVSEC
,
297 GPF_DEVICE_DVSEC_REVID
, dvsec
);
299 dvsec
= (uint8_t *)&(CXLDVSECPortFlexBus
){
300 .cap
= 0x26, /* 68B, IO, Mem, non-MLD */
301 .ctrl
= 0x02, /* IO always enabled */
302 .status
= 0x26, /* same as capabilities */
303 .rcvd_mod_ts_data_phase1
= 0xef, /* WTF? */
305 cxl_component_create_dvsec(cxl_cstate
, CXL2_TYPE3_DEVICE
,
306 PCIE_FLEXBUS_PORT_DVSEC_LENGTH_2_0
,
307 PCIE_FLEXBUS_PORT_DVSEC
,
308 PCIE_FLEXBUS_PORT_DVSEC_REVID_2_0
, dvsec
);
311 static void hdm_decoder_commit(CXLType3Dev
*ct3d
, int which
)
313 ComponentRegisters
*cregs
= &ct3d
->cxl_cstate
.crb
;
314 uint32_t *cache_mem
= cregs
->cache_mem_registers
;
318 /* TODO: Sanity checks that the decoder is possible */
319 ARRAY_FIELD_DP32(cache_mem
, CXL_HDM_DECODER0_CTRL
, COMMIT
, 0);
320 ARRAY_FIELD_DP32(cache_mem
, CXL_HDM_DECODER0_CTRL
, ERR
, 0);
322 ARRAY_FIELD_DP32(cache_mem
, CXL_HDM_DECODER0_CTRL
, COMMITTED
, 1);
325 static void ct3d_reg_write(void *opaque
, hwaddr offset
, uint64_t value
,
328 CXLComponentState
*cxl_cstate
= opaque
;
329 ComponentRegisters
*cregs
= &cxl_cstate
->crb
;
330 CXLType3Dev
*ct3d
= container_of(cxl_cstate
, CXLType3Dev
, cxl_cstate
);
331 uint32_t *cache_mem
= cregs
->cache_mem_registers
;
332 bool should_commit
= false;
336 g_assert(offset
< CXL2_COMPONENT_CM_REGION_SIZE
);
339 case A_CXL_HDM_DECODER0_CTRL
:
340 should_commit
= FIELD_EX32(value
, CXL_HDM_DECODER0_CTRL
, COMMIT
);
347 stl_le_p((uint8_t *)cache_mem
+ offset
, value
);
349 hdm_decoder_commit(ct3d
, which_hdm
);
353 static bool cxl_setup_memory(CXLType3Dev
*ct3d
, Error
**errp
)
355 DeviceState
*ds
= DEVICE(ct3d
);
359 if (!ct3d
->hostmem
) {
360 error_setg(errp
, "memdev property must be set");
364 mr
= host_memory_backend_get_memory(ct3d
->hostmem
);
366 error_setg(errp
, "memdev property must be set");
369 memory_region_set_nonvolatile(mr
, true);
370 memory_region_set_enabled(mr
, true);
371 host_memory_backend_set_mapped(ct3d
->hostmem
, true);
374 name
= g_strdup_printf("cxl-type3-dpa-space:%s", ds
->id
);
376 name
= g_strdup("cxl-type3-dpa-space");
378 address_space_init(&ct3d
->hostmem_as
, mr
, name
);
381 ct3d
->cxl_dstate
.pmem_size
= ct3d
->hostmem
->size
;
384 error_setg(errp
, "lsa property must be set");
391 static DOEProtocol doe_cdat_prot
[] = {
392 { CXL_VENDOR_ID
, CXL_DOE_TABLE_ACCESS
, cxl_doe_cdat_rsp
},
396 static void ct3_realize(PCIDevice
*pci_dev
, Error
**errp
)
398 CXLType3Dev
*ct3d
= CXL_TYPE3(pci_dev
);
399 CXLComponentState
*cxl_cstate
= &ct3d
->cxl_cstate
;
400 ComponentRegisters
*regs
= &cxl_cstate
->crb
;
401 MemoryRegion
*mr
= ®s
->component_registers
;
402 uint8_t *pci_conf
= pci_dev
->config
;
403 unsigned short msix_num
= 1;
406 if (!cxl_setup_memory(ct3d
, errp
)) {
410 pci_config_set_prog_interface(pci_conf
, 0x10);
411 pci_config_set_class(pci_conf
, PCI_CLASS_MEMORY_CXL
);
413 pcie_endpoint_cap_init(pci_dev
, 0x80);
414 if (ct3d
->sn
!= UI64_NULL
) {
415 pcie_dev_ser_num_init(pci_dev
, 0x100, ct3d
->sn
);
416 cxl_cstate
->dvsec_offset
= 0x100 + 0x0c;
418 cxl_cstate
->dvsec_offset
= 0x100;
421 ct3d
->cxl_cstate
.pdev
= pci_dev
;
424 regs
->special_ops
= g_new0(MemoryRegionOps
, 1);
425 regs
->special_ops
->write
= ct3d_reg_write
;
427 cxl_component_register_block_init(OBJECT(pci_dev
), cxl_cstate
,
431 pci_dev
, CXL_COMPONENT_REG_BAR_IDX
,
432 PCI_BASE_ADDRESS_SPACE_MEMORY
| PCI_BASE_ADDRESS_MEM_TYPE_64
, mr
);
434 cxl_device_register_block_init(OBJECT(pci_dev
), &ct3d
->cxl_dstate
);
435 pci_register_bar(pci_dev
, CXL_DEVICE_REG_BAR_IDX
,
436 PCI_BASE_ADDRESS_SPACE_MEMORY
|
437 PCI_BASE_ADDRESS_MEM_TYPE_64
,
438 &ct3d
->cxl_dstate
.device_registers
);
440 /* MSI(-X) Initailization */
441 msix_init_exclusive_bar(pci_dev
, msix_num
, 4, NULL
);
442 for (i
= 0; i
< msix_num
; i
++) {
443 msix_vector_use(pci_dev
, i
);
446 /* DOE Initailization */
447 pcie_doe_init(pci_dev
, &ct3d
->doe_cdat
, 0x190, doe_cdat_prot
, true, 0);
449 cxl_cstate
->cdat
.build_cdat_table
= ct3_build_cdat_table
;
450 cxl_cstate
->cdat
.free_cdat_table
= ct3_free_cdat_table
;
451 cxl_cstate
->cdat
.private = ct3d
;
452 cxl_doe_cdat_init(cxl_cstate
, errp
);
455 static void ct3_exit(PCIDevice
*pci_dev
)
457 CXLType3Dev
*ct3d
= CXL_TYPE3(pci_dev
);
458 CXLComponentState
*cxl_cstate
= &ct3d
->cxl_cstate
;
459 ComponentRegisters
*regs
= &cxl_cstate
->crb
;
461 cxl_doe_cdat_release(cxl_cstate
);
462 g_free(regs
->special_ops
);
463 address_space_destroy(&ct3d
->hostmem_as
);
466 /* TODO: Support multiple HDM decoders and DPA skip */
467 static bool cxl_type3_dpa(CXLType3Dev
*ct3d
, hwaddr host_addr
, uint64_t *dpa
)
469 uint32_t *cache_mem
= ct3d
->cxl_cstate
.crb
.cache_mem_registers
;
470 uint64_t decoder_base
, decoder_size
, hpa_offset
;
474 decoder_base
= (((uint64_t)cache_mem
[R_CXL_HDM_DECODER0_BASE_HI
] << 32) |
475 cache_mem
[R_CXL_HDM_DECODER0_BASE_LO
]);
476 if ((uint64_t)host_addr
< decoder_base
) {
480 hpa_offset
= (uint64_t)host_addr
- decoder_base
;
482 decoder_size
= ((uint64_t)cache_mem
[R_CXL_HDM_DECODER0_SIZE_HI
] << 32) |
483 cache_mem
[R_CXL_HDM_DECODER0_SIZE_LO
];
484 if (hpa_offset
>= decoder_size
) {
488 hdm0_ctrl
= cache_mem
[R_CXL_HDM_DECODER0_CTRL
];
489 iw
= FIELD_EX32(hdm0_ctrl
, CXL_HDM_DECODER0_CTRL
, IW
);
490 ig
= FIELD_EX32(hdm0_ctrl
, CXL_HDM_DECODER0_CTRL
, IG
);
492 *dpa
= (MAKE_64BIT_MASK(0, 8 + ig
) & hpa_offset
) |
493 ((MAKE_64BIT_MASK(8 + ig
+ iw
, 64 - 8 - ig
- iw
) & hpa_offset
) >> iw
);
498 MemTxResult
cxl_type3_read(PCIDevice
*d
, hwaddr host_addr
, uint64_t *data
,
499 unsigned size
, MemTxAttrs attrs
)
501 CXLType3Dev
*ct3d
= CXL_TYPE3(d
);
505 /* TODO support volatile region */
506 mr
= host_memory_backend_get_memory(ct3d
->hostmem
);
511 if (!cxl_type3_dpa(ct3d
, host_addr
, &dpa_offset
)) {
515 if (dpa_offset
> int128_get64(mr
->size
)) {
519 return address_space_read(&ct3d
->hostmem_as
, dpa_offset
, attrs
, data
, size
);
522 MemTxResult
cxl_type3_write(PCIDevice
*d
, hwaddr host_addr
, uint64_t data
,
523 unsigned size
, MemTxAttrs attrs
)
525 CXLType3Dev
*ct3d
= CXL_TYPE3(d
);
529 mr
= host_memory_backend_get_memory(ct3d
->hostmem
);
534 if (!cxl_type3_dpa(ct3d
, host_addr
, &dpa_offset
)) {
538 if (dpa_offset
> int128_get64(mr
->size
)) {
541 return address_space_write(&ct3d
->hostmem_as
, dpa_offset
, attrs
,
545 static void ct3d_reset(DeviceState
*dev
)
547 CXLType3Dev
*ct3d
= CXL_TYPE3(dev
);
548 uint32_t *reg_state
= ct3d
->cxl_cstate
.crb
.cache_mem_registers
;
549 uint32_t *write_msk
= ct3d
->cxl_cstate
.crb
.cache_mem_regs_write_mask
;
551 cxl_component_register_init_common(reg_state
, write_msk
, CXL2_TYPE3_DEVICE
);
552 cxl_device_register_init_common(&ct3d
->cxl_dstate
);
555 static Property ct3_props
[] = {
556 DEFINE_PROP_LINK("memdev", CXLType3Dev
, hostmem
, TYPE_MEMORY_BACKEND
,
557 HostMemoryBackend
*),
558 DEFINE_PROP_LINK("lsa", CXLType3Dev
, lsa
, TYPE_MEMORY_BACKEND
,
559 HostMemoryBackend
*),
560 DEFINE_PROP_UINT64("sn", CXLType3Dev
, sn
, UI64_NULL
),
561 DEFINE_PROP_STRING("cdat", CXLType3Dev
, cxl_cstate
.cdat
.filename
),
562 DEFINE_PROP_END_OF_LIST(),
565 static uint64_t get_lsa_size(CXLType3Dev
*ct3d
)
569 mr
= host_memory_backend_get_memory(ct3d
->lsa
);
570 return memory_region_size(mr
);
573 static void validate_lsa_access(MemoryRegion
*mr
, uint64_t size
,
576 assert(offset
+ size
<= memory_region_size(mr
));
577 assert(offset
+ size
> offset
);
580 static uint64_t get_lsa(CXLType3Dev
*ct3d
, void *buf
, uint64_t size
,
586 mr
= host_memory_backend_get_memory(ct3d
->lsa
);
587 validate_lsa_access(mr
, size
, offset
);
589 lsa
= memory_region_get_ram_ptr(mr
) + offset
;
590 memcpy(buf
, lsa
, size
);
595 static void set_lsa(CXLType3Dev
*ct3d
, const void *buf
, uint64_t size
,
601 mr
= host_memory_backend_get_memory(ct3d
->lsa
);
602 validate_lsa_access(mr
, size
, offset
);
604 lsa
= memory_region_get_ram_ptr(mr
) + offset
;
605 memcpy(lsa
, buf
, size
);
606 memory_region_set_dirty(mr
, offset
, size
);
609 * Just like the PMEM, if the guest is not allowed to exit gracefully, label
610 * updates will get lost.
614 static void ct3_class_init(ObjectClass
*oc
, void *data
)
616 DeviceClass
*dc
= DEVICE_CLASS(oc
);
617 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
618 CXLType3Class
*cvc
= CXL_TYPE3_CLASS(oc
);
620 pc
->realize
= ct3_realize
;
622 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
623 pc
->vendor_id
= PCI_VENDOR_ID_INTEL
;
624 pc
->device_id
= 0xd93; /* LVF for now */
627 pc
->config_write
= ct3d_config_write
;
628 pc
->config_read
= ct3d_config_read
;
630 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
631 dc
->desc
= "CXL PMEM Device (Type 3)";
632 dc
->reset
= ct3d_reset
;
633 device_class_set_props(dc
, ct3_props
);
635 cvc
->get_lsa_size
= get_lsa_size
;
636 cvc
->get_lsa
= get_lsa
;
637 cvc
->set_lsa
= set_lsa
;
640 static const TypeInfo ct3d_info
= {
641 .name
= TYPE_CXL_TYPE3
,
642 .parent
= TYPE_PCI_DEVICE
,
643 .class_size
= sizeof(struct CXLType3Class
),
644 .class_init
= ct3_class_init
,
645 .instance_size
= sizeof(CXLType3Dev
),
646 .interfaces
= (InterfaceInfo
[]) {
647 { INTERFACE_CXL_DEVICE
},
648 { INTERFACE_PCIE_DEVICE
},
653 static void ct3d_registers(void)
655 type_register_static(&ct3d_info
);
658 type_init(ct3d_registers
);