1 #include "qemu/osdep.h"
2 #include "hw/block/block.h"
3 #include "sysemu/dma.h"
4 #include "sysemu/block-backend.h"
5 #include "qapi/error.h"
10 uint16_t nvme_check_prinfo(NvmeNamespace
*ns
, uint16_t ctrl
, uint64_t slba
,
13 if ((NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
) == NVME_ID_NS_DPS_TYPE_1
) &&
14 (ctrl
& NVME_RW_PRINFO_PRCHK_REF
) && (slba
& 0xffffffff) != reftag
) {
15 return NVME_INVALID_PROT_INFO
| NVME_DNR
;
21 /* from Linux kernel (crypto/crct10dif_common.c) */
22 static uint16_t crc_t10dif(uint16_t crc
, const unsigned char *buffer
,
27 for (i
= 0; i
< len
; i
++) {
28 crc
= (crc
<< 8) ^ t10_dif_crc_table
[((crc
>> 8) ^ buffer
[i
]) & 0xff];
34 void nvme_dif_pract_generate_dif(NvmeNamespace
*ns
, uint8_t *buf
, size_t len
,
35 uint8_t *mbuf
, size_t mlen
, uint16_t apptag
,
38 uint8_t *end
= buf
+ len
;
39 size_t lsize
= nvme_lsize(ns
);
40 size_t msize
= nvme_msize(ns
);
43 if (!(ns
->id_ns
.dps
& NVME_ID_NS_DPS_FIRST_EIGHT
)) {
44 pil
= nvme_msize(ns
) - sizeof(NvmeDifTuple
);
47 trace_pci_nvme_dif_pract_generate_dif(len
, lsize
, lsize
+ pil
, apptag
,
50 for (; buf
< end
; buf
+= lsize
, mbuf
+= msize
) {
51 NvmeDifTuple
*dif
= (NvmeDifTuple
*)(mbuf
+ pil
);
52 uint16_t crc
= crc_t10dif(0x0, buf
, lsize
);
55 crc
= crc_t10dif(crc
, mbuf
, pil
);
58 dif
->guard
= cpu_to_be16(crc
);
59 dif
->apptag
= cpu_to_be16(apptag
);
60 dif
->reftag
= cpu_to_be32(reftag
);
62 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
) != NVME_ID_NS_DPS_TYPE_3
) {
68 static uint16_t nvme_dif_prchk(NvmeNamespace
*ns
, NvmeDifTuple
*dif
,
69 uint8_t *buf
, uint8_t *mbuf
, size_t pil
,
70 uint16_t ctrl
, uint16_t apptag
,
71 uint16_t appmask
, uint32_t reftag
)
73 switch (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
74 case NVME_ID_NS_DPS_TYPE_3
:
75 if (be32_to_cpu(dif
->reftag
) != 0xffffffff) {
80 case NVME_ID_NS_DPS_TYPE_1
:
81 case NVME_ID_NS_DPS_TYPE_2
:
82 if (be16_to_cpu(dif
->apptag
) != 0xffff) {
86 trace_pci_nvme_dif_prchk_disabled(be16_to_cpu(dif
->apptag
),
87 be32_to_cpu(dif
->reftag
));
92 if (ctrl
& NVME_RW_PRINFO_PRCHK_GUARD
) {
93 uint16_t crc
= crc_t10dif(0x0, buf
, nvme_lsize(ns
));
96 crc
= crc_t10dif(crc
, mbuf
, pil
);
99 trace_pci_nvme_dif_prchk_guard(be16_to_cpu(dif
->guard
), crc
);
101 if (be16_to_cpu(dif
->guard
) != crc
) {
102 return NVME_E2E_GUARD_ERROR
;
106 if (ctrl
& NVME_RW_PRINFO_PRCHK_APP
) {
107 trace_pci_nvme_dif_prchk_apptag(be16_to_cpu(dif
->apptag
), apptag
,
110 if ((be16_to_cpu(dif
->apptag
) & appmask
) != (apptag
& appmask
)) {
111 return NVME_E2E_APP_ERROR
;
115 if (ctrl
& NVME_RW_PRINFO_PRCHK_REF
) {
116 trace_pci_nvme_dif_prchk_reftag(be32_to_cpu(dif
->reftag
), reftag
);
118 if (be32_to_cpu(dif
->reftag
) != reftag
) {
119 return NVME_E2E_REF_ERROR
;
126 uint16_t nvme_dif_check(NvmeNamespace
*ns
, uint8_t *buf
, size_t len
,
127 uint8_t *mbuf
, size_t mlen
, uint16_t ctrl
,
128 uint64_t slba
, uint16_t apptag
,
129 uint16_t appmask
, uint32_t reftag
)
131 uint8_t *end
= buf
+ len
;
132 size_t lsize
= nvme_lsize(ns
);
133 size_t msize
= nvme_msize(ns
);
137 status
= nvme_check_prinfo(ns
, ctrl
, slba
, reftag
);
142 if (!(ns
->id_ns
.dps
& NVME_ID_NS_DPS_FIRST_EIGHT
)) {
143 pil
= nvme_msize(ns
) - sizeof(NvmeDifTuple
);
146 trace_pci_nvme_dif_check(NVME_RW_PRINFO(ctrl
), lsize
+ pil
);
148 for (; buf
< end
; buf
+= lsize
, mbuf
+= msize
) {
149 NvmeDifTuple
*dif
= (NvmeDifTuple
*)(mbuf
+ pil
);
151 status
= nvme_dif_prchk(ns
, dif
, buf
, mbuf
, pil
, ctrl
, apptag
,
157 if (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
) != NVME_ID_NS_DPS_TYPE_3
) {
165 uint16_t nvme_dif_mangle_mdata(NvmeNamespace
*ns
, uint8_t *mbuf
, size_t mlen
,
168 BlockBackend
*blk
= ns
->blkconf
.blk
;
169 BlockDriverState
*bs
= blk_bs(blk
);
171 size_t msize
= nvme_msize(ns
);
172 size_t lsize
= nvme_lsize(ns
);
173 int64_t moffset
= 0, offset
= nvme_l2b(ns
, slba
);
174 uint8_t *mbufp
, *end
;
177 int64_t bytes
= (mlen
/ msize
) * lsize
;
183 if (!(ns
->id_ns
.dps
& NVME_ID_NS_DPS_FIRST_EIGHT
)) {
184 pil
= nvme_msize(ns
) - sizeof(NvmeDifTuple
);
192 ret
= bdrv_block_status(bs
, offset
, bytes
, &pnum
, NULL
, NULL
);
194 error_setg_errno(&err
, -ret
, "unable to get block status");
195 error_report_err(err
);
197 return NVME_INTERNAL_DEV_ERROR
;
200 zeroed
= !!(ret
& BDRV_BLOCK_ZERO
);
202 trace_pci_nvme_block_status(offset
, bytes
, pnum
, ret
, zeroed
);
205 mbufp
= mbuf
+ moffset
;
206 mlen
= (pnum
/ lsize
) * msize
;
209 for (; mbufp
< end
; mbufp
+= msize
) {
210 memset(mbufp
+ pil
, 0xff, sizeof(NvmeDifTuple
));
214 moffset
+= (pnum
/ lsize
) * msize
;
216 } while (pnum
!= bytes
);
221 static void nvme_dif_rw_cb(void *opaque
, int ret
)
223 NvmeBounceContext
*ctx
= opaque
;
224 NvmeRequest
*req
= ctx
->req
;
225 NvmeNamespace
*ns
= req
->ns
;
226 BlockBackend
*blk
= ns
->blkconf
.blk
;
228 trace_pci_nvme_dif_rw_cb(nvme_cid(req
), blk_name(blk
));
230 qemu_iovec_destroy(&ctx
->data
.iov
);
231 g_free(ctx
->data
.bounce
);
233 qemu_iovec_destroy(&ctx
->mdata
.iov
);
234 g_free(ctx
->mdata
.bounce
);
238 nvme_rw_complete_cb(req
, ret
);
241 static void nvme_dif_rw_check_cb(void *opaque
, int ret
)
243 NvmeBounceContext
*ctx
= opaque
;
244 NvmeRequest
*req
= ctx
->req
;
245 NvmeNamespace
*ns
= req
->ns
;
246 NvmeCtrl
*n
= nvme_ctrl(req
);
247 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
248 uint64_t slba
= le64_to_cpu(rw
->slba
);
249 uint16_t ctrl
= le16_to_cpu(rw
->control
);
250 uint16_t apptag
= le16_to_cpu(rw
->apptag
);
251 uint16_t appmask
= le16_to_cpu(rw
->appmask
);
252 uint32_t reftag
= le32_to_cpu(rw
->reftag
);
255 trace_pci_nvme_dif_rw_check_cb(nvme_cid(req
), NVME_RW_PRINFO(ctrl
), apptag
,
262 status
= nvme_dif_mangle_mdata(ns
, ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
,
265 req
->status
= status
;
269 status
= nvme_dif_check(ns
, ctx
->data
.bounce
, ctx
->data
.iov
.size
,
270 ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
, ctrl
,
271 slba
, apptag
, appmask
, reftag
);
273 req
->status
= status
;
277 status
= nvme_bounce_data(n
, ctx
->data
.bounce
, ctx
->data
.iov
.size
,
278 NVME_TX_DIRECTION_FROM_DEVICE
, req
);
280 req
->status
= status
;
284 if (ctrl
& NVME_RW_PRINFO_PRACT
&& nvme_msize(ns
) == 8) {
288 status
= nvme_bounce_mdata(n
, ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
,
289 NVME_TX_DIRECTION_FROM_DEVICE
, req
);
291 req
->status
= status
;
295 nvme_dif_rw_cb(ctx
, ret
);
298 static void nvme_dif_rw_mdata_in_cb(void *opaque
, int ret
)
300 NvmeBounceContext
*ctx
= opaque
;
301 NvmeRequest
*req
= ctx
->req
;
302 NvmeNamespace
*ns
= req
->ns
;
303 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
304 uint64_t slba
= le64_to_cpu(rw
->slba
);
305 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
306 size_t mlen
= nvme_m2b(ns
, nlb
);
307 uint64_t offset
= ns
->mdata_offset
+ nvme_m2b(ns
, slba
);
308 BlockBackend
*blk
= ns
->blkconf
.blk
;
310 trace_pci_nvme_dif_rw_mdata_in_cb(nvme_cid(req
), blk_name(blk
));
316 ctx
->mdata
.bounce
= g_malloc(mlen
);
318 qemu_iovec_reset(&ctx
->mdata
.iov
);
319 qemu_iovec_add(&ctx
->mdata
.iov
, ctx
->mdata
.bounce
, mlen
);
321 req
->aiocb
= blk_aio_preadv(blk
, offset
, &ctx
->mdata
.iov
, 0,
322 nvme_dif_rw_check_cb
, ctx
);
326 nvme_dif_rw_cb(ctx
, ret
);
329 static void nvme_dif_rw_mdata_out_cb(void *opaque
, int ret
)
331 NvmeBounceContext
*ctx
= opaque
;
332 NvmeRequest
*req
= ctx
->req
;
333 NvmeNamespace
*ns
= req
->ns
;
334 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
335 uint64_t slba
= le64_to_cpu(rw
->slba
);
336 uint64_t offset
= ns
->mdata_offset
+ nvme_m2b(ns
, slba
);
337 BlockBackend
*blk
= ns
->blkconf
.blk
;
339 trace_pci_nvme_dif_rw_mdata_out_cb(nvme_cid(req
), blk_name(blk
));
345 req
->aiocb
= blk_aio_pwritev(blk
, offset
, &ctx
->mdata
.iov
, 0,
346 nvme_dif_rw_cb
, ctx
);
350 nvme_dif_rw_cb(ctx
, ret
);
353 uint16_t nvme_dif_rw(NvmeCtrl
*n
, NvmeRequest
*req
)
355 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
356 NvmeNamespace
*ns
= req
->ns
;
357 BlockBackend
*blk
= ns
->blkconf
.blk
;
358 bool wrz
= rw
->opcode
== NVME_CMD_WRITE_ZEROES
;
359 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
360 uint64_t slba
= le64_to_cpu(rw
->slba
);
361 size_t len
= nvme_l2b(ns
, nlb
);
362 size_t mlen
= nvme_m2b(ns
, nlb
);
363 size_t mapped_len
= len
;
364 int64_t offset
= nvme_l2b(ns
, slba
);
365 uint16_t ctrl
= le16_to_cpu(rw
->control
);
366 uint16_t apptag
= le16_to_cpu(rw
->apptag
);
367 uint16_t appmask
= le16_to_cpu(rw
->appmask
);
368 uint32_t reftag
= le32_to_cpu(rw
->reftag
);
369 bool pract
= !!(ctrl
& NVME_RW_PRINFO_PRACT
);
370 NvmeBounceContext
*ctx
;
373 trace_pci_nvme_dif_rw(pract
, NVME_RW_PRINFO(ctrl
));
375 ctx
= g_new0(NvmeBounceContext
, 1);
379 BdrvRequestFlags flags
= BDRV_REQ_MAY_UNMAP
;
381 if (ctrl
& NVME_RW_PRINFO_PRCHK_MASK
) {
382 status
= NVME_INVALID_PROT_INFO
| NVME_DNR
;
388 size_t msize
= nvme_msize(ns
);
389 int16_t pil
= msize
- sizeof(NvmeDifTuple
);
391 status
= nvme_check_prinfo(ns
, ctrl
, slba
, reftag
);
398 ctx
->mdata
.bounce
= g_malloc0(mlen
);
400 qemu_iovec_init(&ctx
->mdata
.iov
, 1);
401 qemu_iovec_add(&ctx
->mdata
.iov
, ctx
->mdata
.bounce
, mlen
);
403 mbuf
= ctx
->mdata
.bounce
;
406 if (ns
->id_ns
.dps
& NVME_ID_NS_DPS_FIRST_EIGHT
) {
410 for (; mbuf
< end
; mbuf
+= msize
) {
411 NvmeDifTuple
*dif
= (NvmeDifTuple
*)(mbuf
+ pil
);
413 dif
->apptag
= cpu_to_be16(apptag
);
414 dif
->reftag
= cpu_to_be32(reftag
);
416 switch (NVME_ID_NS_DPS_TYPE(ns
->id_ns
.dps
)) {
417 case NVME_ID_NS_DPS_TYPE_1
:
418 case NVME_ID_NS_DPS_TYPE_2
:
424 req
->aiocb
= blk_aio_pwrite_zeroes(blk
, offset
, len
, flags
,
425 nvme_dif_rw_mdata_out_cb
, ctx
);
426 return NVME_NO_COMPLETE
;
429 if (nvme_ns_ext(ns
) && !(pract
&& nvme_msize(ns
) == 8)) {
433 status
= nvme_map_dptr(n
, &req
->sg
, mapped_len
, &req
->cmd
);
438 ctx
->data
.bounce
= g_malloc(len
);
440 qemu_iovec_init(&ctx
->data
.iov
, 1);
441 qemu_iovec_add(&ctx
->data
.iov
, ctx
->data
.bounce
, len
);
443 if (req
->cmd
.opcode
== NVME_CMD_READ
) {
444 block_acct_start(blk_get_stats(blk
), &req
->acct
, ctx
->data
.iov
.size
,
447 req
->aiocb
= blk_aio_preadv(ns
->blkconf
.blk
, offset
, &ctx
->data
.iov
, 0,
448 nvme_dif_rw_mdata_in_cb
, ctx
);
449 return NVME_NO_COMPLETE
;
452 status
= nvme_bounce_data(n
, ctx
->data
.bounce
, ctx
->data
.iov
.size
,
453 NVME_TX_DIRECTION_TO_DEVICE
, req
);
458 ctx
->mdata
.bounce
= g_malloc(mlen
);
460 qemu_iovec_init(&ctx
->mdata
.iov
, 1);
461 qemu_iovec_add(&ctx
->mdata
.iov
, ctx
->mdata
.bounce
, mlen
);
463 if (!(pract
&& nvme_msize(ns
) == 8)) {
464 status
= nvme_bounce_mdata(n
, ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
,
465 NVME_TX_DIRECTION_TO_DEVICE
, req
);
471 status
= nvme_check_prinfo(ns
, ctrl
, slba
, reftag
);
477 /* splice generated protection information into the buffer */
478 nvme_dif_pract_generate_dif(ns
, ctx
->data
.bounce
, ctx
->data
.iov
.size
,
479 ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
,
482 status
= nvme_dif_check(ns
, ctx
->data
.bounce
, ctx
->data
.iov
.size
,
483 ctx
->mdata
.bounce
, ctx
->mdata
.iov
.size
, ctrl
,
484 slba
, apptag
, appmask
, reftag
);
490 block_acct_start(blk_get_stats(blk
), &req
->acct
, ctx
->data
.iov
.size
,
493 req
->aiocb
= blk_aio_pwritev(ns
->blkconf
.blk
, offset
, &ctx
->data
.iov
, 0,
494 nvme_dif_rw_mdata_out_cb
, ctx
);
496 return NVME_NO_COMPLETE
;
499 qemu_iovec_destroy(&ctx
->data
.iov
);
500 g_free(ctx
->data
.bounce
);
502 qemu_iovec_destroy(&ctx
->mdata
.iov
);
503 g_free(ctx
->mdata
.bounce
);