1 // SPDX-License-Identifier: GPL-2.0
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
5 * Work Requests exploiting Infiniband API
7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
8 * are submitted to either RC SQ or RC RQ respectively
9 * (reliably connected send/receive queue)
10 * and become work queue entries (WQEs).
11 * While an SQ WR/WQE is pending, we track it until transmission completion.
12 * Through a send or receive completion queue (CQ) respectively,
13 * we get completion queue entries (CQEs) [aka work completions (WCs)].
14 * Since the CQ callback is called from IRQ context, we split work by using
15 * bottom halves implemented by tasklets.
17 * SMC uses this to exchange LLC (link layer control)
18 * and CDC (connection data control) messages.
20 * Copyright IBM Corp. 2016
22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com>
25 #include <linux/atomic.h>
26 #include <linux/hashtable.h>
27 #include <linux/wait.h>
28 #include <rdma/ib_verbs.h>
29 #include <asm/div64.h>
34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */
36 #define SMC_WR_RX_HASH_BITS 4
37 static DEFINE_HASHTABLE(smc_wr_rx_hash
, SMC_WR_RX_HASH_BITS
);
38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock
);
40 struct smc_wr_tx_pend
{ /* control data for a pending send request */
41 u64 wr_id
; /* work request id sent */
42 smc_wr_tx_handler handler
;
43 enum ib_wc_status wc_status
; /* CQE status */
44 struct smc_link
*link
;
46 struct smc_wr_tx_pend_priv priv
;
49 /******************************** send queue *********************************/
51 /*------------------------------- completion --------------------------------*/
53 static inline int smc_wr_tx_find_pending_index(struct smc_link
*link
, u64 wr_id
)
57 for (i
= 0; i
< link
->wr_tx_cnt
; i
++) {
58 if (link
->wr_tx_pends
[i
].wr_id
== wr_id
)
61 return link
->wr_tx_cnt
;
64 static inline void smc_wr_tx_process_cqe(struct ib_wc
*wc
)
66 struct smc_wr_tx_pend pnd_snd
;
67 struct smc_link
*link
;
71 link
= wc
->qp
->qp_context
;
73 if (wc
->opcode
== IB_WC_REG_MR
) {
75 link
->wr_reg_state
= FAILED
;
77 link
->wr_reg_state
= CONFIRMED
;
78 wake_up(&link
->wr_reg_wait
);
82 pnd_snd_idx
= smc_wr_tx_find_pending_index(link
, wc
->wr_id
);
83 if (pnd_snd_idx
== link
->wr_tx_cnt
)
85 link
->wr_tx_pends
[pnd_snd_idx
].wc_status
= wc
->status
;
86 memcpy(&pnd_snd
, &link
->wr_tx_pends
[pnd_snd_idx
], sizeof(pnd_snd
));
87 /* clear the full struct smc_wr_tx_pend including .priv */
88 memset(&link
->wr_tx_pends
[pnd_snd_idx
], 0,
89 sizeof(link
->wr_tx_pends
[pnd_snd_idx
]));
90 memset(&link
->wr_tx_bufs
[pnd_snd_idx
], 0,
91 sizeof(link
->wr_tx_bufs
[pnd_snd_idx
]));
92 if (!test_and_clear_bit(pnd_snd_idx
, link
->wr_tx_mask
))
95 for_each_set_bit(i
, link
->wr_tx_mask
, link
->wr_tx_cnt
) {
96 /* clear full struct smc_wr_tx_pend including .priv */
97 memset(&link
->wr_tx_pends
[i
], 0,
98 sizeof(link
->wr_tx_pends
[i
]));
99 memset(&link
->wr_tx_bufs
[i
], 0,
100 sizeof(link
->wr_tx_bufs
[i
]));
101 clear_bit(i
, link
->wr_tx_mask
);
103 /* terminate connections of this link group abnormally */
104 smc_lgr_terminate(smc_get_lgr(link
));
107 pnd_snd
.handler(&pnd_snd
.priv
, link
, wc
->status
);
108 wake_up(&link
->wr_tx_wait
);
111 static void smc_wr_tx_tasklet_fn(unsigned long data
)
113 struct smc_ib_device
*dev
= (struct smc_ib_device
*)data
;
114 struct ib_wc wc
[SMC_WR_MAX_POLL_CQE
];
121 memset(&wc
, 0, sizeof(wc
));
122 rc
= ib_poll_cq(dev
->roce_cq_send
, SMC_WR_MAX_POLL_CQE
, wc
);
124 ib_req_notify_cq(dev
->roce_cq_send
,
126 IB_CQ_REPORT_MISSED_EVENTS
);
130 for (i
= 0; i
< rc
; i
++)
131 smc_wr_tx_process_cqe(&wc
[i
]);
137 void smc_wr_tx_cq_handler(struct ib_cq
*ib_cq
, void *cq_context
)
139 struct smc_ib_device
*dev
= (struct smc_ib_device
*)cq_context
;
141 tasklet_schedule(&dev
->send_tasklet
);
144 /*---------------------------- request submission ---------------------------*/
146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link
*link
, u32
*idx
)
148 *idx
= link
->wr_tx_cnt
;
149 for_each_clear_bit(*idx
, link
->wr_tx_mask
, link
->wr_tx_cnt
) {
150 if (!test_and_set_bit(*idx
, link
->wr_tx_mask
))
153 *idx
= link
->wr_tx_cnt
;
158 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
159 * and sets info for pending transmit tracking
160 * @link: Pointer to smc_link used to later send the message.
161 * @handler: Send completion handler function pointer.
162 * @wr_buf: Out value returns pointer to message buffer.
163 * @wr_pend_priv: Out value returns pointer serving as handler context.
165 * Return: 0 on success, or -errno on error.
167 int smc_wr_tx_get_free_slot(struct smc_link
*link
,
168 smc_wr_tx_handler handler
,
169 struct smc_wr_buf
**wr_buf
,
170 struct smc_wr_tx_pend_priv
**wr_pend_priv
)
172 struct smc_wr_tx_pend
*wr_pend
;
173 u32 idx
= link
->wr_tx_cnt
;
174 struct ib_send_wr
*wr_ib
;
179 *wr_pend_priv
= NULL
;
181 rc
= smc_wr_tx_get_free_slot_index(link
, &idx
);
185 rc
= wait_event_timeout(
187 link
->state
== SMC_LNK_INACTIVE
||
188 (smc_wr_tx_get_free_slot_index(link
, &idx
) != -EBUSY
),
189 SMC_WR_TX_WAIT_FREE_SLOT_TIME
);
191 /* timeout - terminate connections */
192 smc_lgr_terminate(smc_get_lgr(link
));
195 if (idx
== link
->wr_tx_cnt
)
198 wr_id
= smc_wr_tx_get_next_wr_id(link
);
199 wr_pend
= &link
->wr_tx_pends
[idx
];
200 wr_pend
->wr_id
= wr_id
;
201 wr_pend
->handler
= handler
;
202 wr_pend
->link
= link
;
204 wr_ib
= &link
->wr_tx_ibs
[idx
];
205 wr_ib
->wr_id
= wr_id
;
206 *wr_buf
= &link
->wr_tx_bufs
[idx
];
207 *wr_pend_priv
= &wr_pend
->priv
;
211 int smc_wr_tx_put_slot(struct smc_link
*link
,
212 struct smc_wr_tx_pend_priv
*wr_pend_priv
)
214 struct smc_wr_tx_pend
*pend
;
216 pend
= container_of(wr_pend_priv
, struct smc_wr_tx_pend
, priv
);
217 if (pend
->idx
< link
->wr_tx_cnt
) {
218 /* clear the full struct smc_wr_tx_pend including .priv */
219 memset(&link
->wr_tx_pends
[pend
->idx
], 0,
220 sizeof(link
->wr_tx_pends
[pend
->idx
]));
221 memset(&link
->wr_tx_bufs
[pend
->idx
], 0,
222 sizeof(link
->wr_tx_bufs
[pend
->idx
]));
223 test_and_clear_bit(pend
->idx
, link
->wr_tx_mask
);
230 /* Send prepared WR slot via ib_post_send.
231 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
233 int smc_wr_tx_send(struct smc_link
*link
, struct smc_wr_tx_pend_priv
*priv
)
235 struct smc_wr_tx_pend
*pend
;
238 ib_req_notify_cq(link
->smcibdev
->roce_cq_send
,
239 IB_CQ_NEXT_COMP
| IB_CQ_REPORT_MISSED_EVENTS
);
240 pend
= container_of(priv
, struct smc_wr_tx_pend
, priv
);
241 rc
= ib_post_send(link
->roce_qp
, &link
->wr_tx_ibs
[pend
->idx
], NULL
);
243 smc_wr_tx_put_slot(link
, priv
);
244 smc_lgr_terminate(smc_get_lgr(link
));
249 /* Register a memory region and wait for result. */
250 int smc_wr_reg_send(struct smc_link
*link
, struct ib_mr
*mr
)
254 ib_req_notify_cq(link
->smcibdev
->roce_cq_send
,
255 IB_CQ_NEXT_COMP
| IB_CQ_REPORT_MISSED_EVENTS
);
256 link
->wr_reg_state
= POSTED
;
257 link
->wr_reg
.wr
.wr_id
= (u64
)(uintptr_t)mr
;
258 link
->wr_reg
.mr
= mr
;
259 link
->wr_reg
.key
= mr
->rkey
;
260 rc
= ib_post_send(link
->roce_qp
, &link
->wr_reg
.wr
, NULL
);
264 rc
= wait_event_interruptible_timeout(link
->wr_reg_wait
,
265 (link
->wr_reg_state
!= POSTED
),
266 SMC_WR_REG_MR_WAIT_TIME
);
268 /* timeout - terminate connections */
269 smc_lgr_terminate(smc_get_lgr(link
));
272 if (rc
== -ERESTARTSYS
)
274 switch (link
->wr_reg_state
) {
288 void smc_wr_tx_dismiss_slots(struct smc_link
*link
, u8 wr_tx_hdr_type
,
289 smc_wr_tx_filter filter
,
290 smc_wr_tx_dismisser dismisser
,
293 struct smc_wr_tx_pend_priv
*tx_pend
;
294 struct smc_wr_rx_hdr
*wr_tx
;
297 for_each_set_bit(i
, link
->wr_tx_mask
, link
->wr_tx_cnt
) {
298 wr_tx
= (struct smc_wr_rx_hdr
*)&link
->wr_tx_bufs
[i
];
299 if (wr_tx
->type
!= wr_tx_hdr_type
)
301 tx_pend
= &link
->wr_tx_pends
[i
].priv
;
302 if (filter(tx_pend
, data
))
307 /****************************** receive queue ********************************/
309 int smc_wr_rx_register_handler(struct smc_wr_rx_handler
*handler
)
311 struct smc_wr_rx_handler
*h_iter
;
314 spin_lock(&smc_wr_rx_hash_lock
);
315 hash_for_each_possible(smc_wr_rx_hash
, h_iter
, list
, handler
->type
) {
316 if (h_iter
->type
== handler
->type
) {
321 hash_add(smc_wr_rx_hash
, &handler
->list
, handler
->type
);
323 spin_unlock(&smc_wr_rx_hash_lock
);
327 /* Demultiplex a received work request based on the message type to its handler.
328 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
329 * and not being modified any more afterwards so we don't need to lock it.
331 static inline void smc_wr_rx_demultiplex(struct ib_wc
*wc
)
333 struct smc_link
*link
= (struct smc_link
*)wc
->qp
->qp_context
;
334 struct smc_wr_rx_handler
*handler
;
335 struct smc_wr_rx_hdr
*wr_rx
;
339 if (wc
->byte_len
< sizeof(*wr_rx
))
340 return; /* short message */
341 temp_wr_id
= wc
->wr_id
;
342 index
= do_div(temp_wr_id
, link
->wr_rx_cnt
);
343 wr_rx
= (struct smc_wr_rx_hdr
*)&link
->wr_rx_bufs
[index
];
344 hash_for_each_possible(smc_wr_rx_hash
, handler
, list
, wr_rx
->type
) {
345 if (handler
->type
== wr_rx
->type
)
346 handler
->handler(wc
, wr_rx
);
350 static inline void smc_wr_rx_process_cqes(struct ib_wc wc
[], int num
)
352 struct smc_link
*link
;
355 for (i
= 0; i
< num
; i
++) {
356 link
= wc
[i
].qp
->qp_context
;
357 if (wc
[i
].status
== IB_WC_SUCCESS
) {
358 link
->wr_rx_tstamp
= jiffies
;
359 smc_wr_rx_demultiplex(&wc
[i
]);
360 smc_wr_rx_post(link
); /* refill WR RX */
362 /* handle status errors */
363 switch (wc
[i
].status
) {
364 case IB_WC_RETRY_EXC_ERR
:
365 case IB_WC_RNR_RETRY_EXC_ERR
:
366 case IB_WC_WR_FLUSH_ERR
:
367 /* terminate connections of this link group
370 smc_lgr_terminate(smc_get_lgr(link
));
373 smc_wr_rx_post(link
); /* refill WR RX */
380 static void smc_wr_rx_tasklet_fn(unsigned long data
)
382 struct smc_ib_device
*dev
= (struct smc_ib_device
*)data
;
383 struct ib_wc wc
[SMC_WR_MAX_POLL_CQE
];
390 memset(&wc
, 0, sizeof(wc
));
391 rc
= ib_poll_cq(dev
->roce_cq_recv
, SMC_WR_MAX_POLL_CQE
, wc
);
393 ib_req_notify_cq(dev
->roce_cq_recv
,
395 | IB_CQ_REPORT_MISSED_EVENTS
);
399 smc_wr_rx_process_cqes(&wc
[0], rc
);
405 void smc_wr_rx_cq_handler(struct ib_cq
*ib_cq
, void *cq_context
)
407 struct smc_ib_device
*dev
= (struct smc_ib_device
*)cq_context
;
409 tasklet_schedule(&dev
->recv_tasklet
);
412 int smc_wr_rx_post_init(struct smc_link
*link
)
417 for (i
= 0; i
< link
->wr_rx_cnt
; i
++)
418 rc
= smc_wr_rx_post(link
);
422 /***************************** init, exit, misc ******************************/
424 void smc_wr_remember_qp_attr(struct smc_link
*lnk
)
426 struct ib_qp_attr
*attr
= &lnk
->qp_attr
;
427 struct ib_qp_init_attr init_attr
;
429 memset(attr
, 0, sizeof(*attr
));
430 memset(&init_attr
, 0, sizeof(init_attr
));
431 ib_query_qp(lnk
->roce_qp
, attr
,
444 IB_QP_MIN_RNR_TIMER
|
446 IB_QP_PATH_MIG_STATE
|
451 lnk
->wr_tx_cnt
= min_t(size_t, SMC_WR_BUF_CNT
,
452 lnk
->qp_attr
.cap
.max_send_wr
);
453 lnk
->wr_rx_cnt
= min_t(size_t, SMC_WR_BUF_CNT
* 3,
454 lnk
->qp_attr
.cap
.max_recv_wr
);
457 static void smc_wr_init_sge(struct smc_link
*lnk
)
461 for (i
= 0; i
< lnk
->wr_tx_cnt
; i
++) {
462 lnk
->wr_tx_sges
[i
].addr
=
463 lnk
->wr_tx_dma_addr
+ i
* SMC_WR_BUF_SIZE
;
464 lnk
->wr_tx_sges
[i
].length
= SMC_WR_TX_SIZE
;
465 lnk
->wr_tx_sges
[i
].lkey
= lnk
->roce_pd
->local_dma_lkey
;
466 lnk
->wr_tx_ibs
[i
].next
= NULL
;
467 lnk
->wr_tx_ibs
[i
].sg_list
= &lnk
->wr_tx_sges
[i
];
468 lnk
->wr_tx_ibs
[i
].num_sge
= 1;
469 lnk
->wr_tx_ibs
[i
].opcode
= IB_WR_SEND
;
470 lnk
->wr_tx_ibs
[i
].send_flags
=
471 IB_SEND_SIGNALED
| IB_SEND_SOLICITED
;
473 for (i
= 0; i
< lnk
->wr_rx_cnt
; i
++) {
474 lnk
->wr_rx_sges
[i
].addr
=
475 lnk
->wr_rx_dma_addr
+ i
* SMC_WR_BUF_SIZE
;
476 lnk
->wr_rx_sges
[i
].length
= SMC_WR_BUF_SIZE
;
477 lnk
->wr_rx_sges
[i
].lkey
= lnk
->roce_pd
->local_dma_lkey
;
478 lnk
->wr_rx_ibs
[i
].next
= NULL
;
479 lnk
->wr_rx_ibs
[i
].sg_list
= &lnk
->wr_rx_sges
[i
];
480 lnk
->wr_rx_ibs
[i
].num_sge
= 1;
482 lnk
->wr_reg
.wr
.next
= NULL
;
483 lnk
->wr_reg
.wr
.num_sge
= 0;
484 lnk
->wr_reg
.wr
.send_flags
= IB_SEND_SIGNALED
;
485 lnk
->wr_reg
.wr
.opcode
= IB_WR_REG_MR
;
486 lnk
->wr_reg
.access
= IB_ACCESS_LOCAL_WRITE
| IB_ACCESS_REMOTE_WRITE
;
489 void smc_wr_free_link(struct smc_link
*lnk
)
491 struct ib_device
*ibdev
;
493 memset(lnk
->wr_tx_mask
, 0,
494 BITS_TO_LONGS(SMC_WR_BUF_CNT
) * sizeof(*lnk
->wr_tx_mask
));
498 ibdev
= lnk
->smcibdev
->ibdev
;
500 if (lnk
->wr_rx_dma_addr
) {
501 ib_dma_unmap_single(ibdev
, lnk
->wr_rx_dma_addr
,
502 SMC_WR_BUF_SIZE
* lnk
->wr_rx_cnt
,
504 lnk
->wr_rx_dma_addr
= 0;
506 if (lnk
->wr_tx_dma_addr
) {
507 ib_dma_unmap_single(ibdev
, lnk
->wr_tx_dma_addr
,
508 SMC_WR_BUF_SIZE
* lnk
->wr_tx_cnt
,
510 lnk
->wr_tx_dma_addr
= 0;
514 void smc_wr_free_link_mem(struct smc_link
*lnk
)
516 kfree(lnk
->wr_tx_pends
);
517 lnk
->wr_tx_pends
= NULL
;
518 kfree(lnk
->wr_tx_mask
);
519 lnk
->wr_tx_mask
= NULL
;
520 kfree(lnk
->wr_tx_sges
);
521 lnk
->wr_tx_sges
= NULL
;
522 kfree(lnk
->wr_rx_sges
);
523 lnk
->wr_rx_sges
= NULL
;
524 kfree(lnk
->wr_rx_ibs
);
525 lnk
->wr_rx_ibs
= NULL
;
526 kfree(lnk
->wr_tx_ibs
);
527 lnk
->wr_tx_ibs
= NULL
;
528 kfree(lnk
->wr_tx_bufs
);
529 lnk
->wr_tx_bufs
= NULL
;
530 kfree(lnk
->wr_rx_bufs
);
531 lnk
->wr_rx_bufs
= NULL
;
534 int smc_wr_alloc_link_mem(struct smc_link
*link
)
536 /* allocate link related memory */
537 link
->wr_tx_bufs
= kcalloc(SMC_WR_BUF_CNT
, SMC_WR_BUF_SIZE
, GFP_KERNEL
);
538 if (!link
->wr_tx_bufs
)
540 link
->wr_rx_bufs
= kcalloc(SMC_WR_BUF_CNT
* 3, SMC_WR_BUF_SIZE
,
542 if (!link
->wr_rx_bufs
)
543 goto no_mem_wr_tx_bufs
;
544 link
->wr_tx_ibs
= kcalloc(SMC_WR_BUF_CNT
, sizeof(link
->wr_tx_ibs
[0]),
546 if (!link
->wr_tx_ibs
)
547 goto no_mem_wr_rx_bufs
;
548 link
->wr_rx_ibs
= kcalloc(SMC_WR_BUF_CNT
* 3,
549 sizeof(link
->wr_rx_ibs
[0]),
551 if (!link
->wr_rx_ibs
)
552 goto no_mem_wr_tx_ibs
;
553 link
->wr_tx_sges
= kcalloc(SMC_WR_BUF_CNT
, sizeof(link
->wr_tx_sges
[0]),
555 if (!link
->wr_tx_sges
)
556 goto no_mem_wr_rx_ibs
;
557 link
->wr_rx_sges
= kcalloc(SMC_WR_BUF_CNT
* 3,
558 sizeof(link
->wr_rx_sges
[0]),
560 if (!link
->wr_rx_sges
)
561 goto no_mem_wr_tx_sges
;
562 link
->wr_tx_mask
= kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT
),
563 sizeof(*link
->wr_tx_mask
),
565 if (!link
->wr_tx_mask
)
566 goto no_mem_wr_rx_sges
;
567 link
->wr_tx_pends
= kcalloc(SMC_WR_BUF_CNT
,
568 sizeof(link
->wr_tx_pends
[0]),
570 if (!link
->wr_tx_pends
)
571 goto no_mem_wr_tx_mask
;
575 kfree(link
->wr_tx_mask
);
577 kfree(link
->wr_rx_sges
);
579 kfree(link
->wr_tx_sges
);
581 kfree(link
->wr_rx_ibs
);
583 kfree(link
->wr_tx_ibs
);
585 kfree(link
->wr_rx_bufs
);
587 kfree(link
->wr_tx_bufs
);
592 void smc_wr_remove_dev(struct smc_ib_device
*smcibdev
)
594 tasklet_kill(&smcibdev
->recv_tasklet
);
595 tasklet_kill(&smcibdev
->send_tasklet
);
598 void smc_wr_add_dev(struct smc_ib_device
*smcibdev
)
600 tasklet_init(&smcibdev
->recv_tasklet
, smc_wr_rx_tasklet_fn
,
601 (unsigned long)smcibdev
);
602 tasklet_init(&smcibdev
->send_tasklet
, smc_wr_tx_tasklet_fn
,
603 (unsigned long)smcibdev
);
606 int smc_wr_create_link(struct smc_link
*lnk
)
608 struct ib_device
*ibdev
= lnk
->smcibdev
->ibdev
;
611 smc_wr_tx_set_wr_id(&lnk
->wr_tx_id
, 0);
613 lnk
->wr_rx_dma_addr
= ib_dma_map_single(
614 ibdev
, lnk
->wr_rx_bufs
, SMC_WR_BUF_SIZE
* lnk
->wr_rx_cnt
,
616 if (ib_dma_mapping_error(ibdev
, lnk
->wr_rx_dma_addr
)) {
617 lnk
->wr_rx_dma_addr
= 0;
621 lnk
->wr_tx_dma_addr
= ib_dma_map_single(
622 ibdev
, lnk
->wr_tx_bufs
, SMC_WR_BUF_SIZE
* lnk
->wr_tx_cnt
,
624 if (ib_dma_mapping_error(ibdev
, lnk
->wr_tx_dma_addr
)) {
628 smc_wr_init_sge(lnk
);
629 memset(lnk
->wr_tx_mask
, 0,
630 BITS_TO_LONGS(SMC_WR_BUF_CNT
) * sizeof(*lnk
->wr_tx_mask
));
631 init_waitqueue_head(&lnk
->wr_tx_wait
);
632 init_waitqueue_head(&lnk
->wr_reg_wait
);
636 ib_dma_unmap_single(ibdev
, lnk
->wr_rx_dma_addr
,
637 SMC_WR_BUF_SIZE
* lnk
->wr_rx_cnt
,
639 lnk
->wr_rx_dma_addr
= 0;