2 * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4 * Copyright (c) 2004 Intel Corporation. All rights reserved.
5 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
6 * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/export.h>
42 #include <linux/string.h>
43 #include <linux/slab.h>
45 #include <linux/in6.h>
46 #include <net/addrconf.h>
48 #include <rdma/ib_verbs.h>
49 #include <rdma/ib_cache.h>
50 #include <rdma/ib_addr.h>
52 #include "core_priv.h"
54 static const char * const ib_events
[] = {
55 [IB_EVENT_CQ_ERR
] = "CQ error",
56 [IB_EVENT_QP_FATAL
] = "QP fatal error",
57 [IB_EVENT_QP_REQ_ERR
] = "QP request error",
58 [IB_EVENT_QP_ACCESS_ERR
] = "QP access error",
59 [IB_EVENT_COMM_EST
] = "communication established",
60 [IB_EVENT_SQ_DRAINED
] = "send queue drained",
61 [IB_EVENT_PATH_MIG
] = "path migration successful",
62 [IB_EVENT_PATH_MIG_ERR
] = "path migration error",
63 [IB_EVENT_DEVICE_FATAL
] = "device fatal error",
64 [IB_EVENT_PORT_ACTIVE
] = "port active",
65 [IB_EVENT_PORT_ERR
] = "port error",
66 [IB_EVENT_LID_CHANGE
] = "LID change",
67 [IB_EVENT_PKEY_CHANGE
] = "P_key change",
68 [IB_EVENT_SM_CHANGE
] = "SM change",
69 [IB_EVENT_SRQ_ERR
] = "SRQ error",
70 [IB_EVENT_SRQ_LIMIT_REACHED
] = "SRQ limit reached",
71 [IB_EVENT_QP_LAST_WQE_REACHED
] = "last WQE reached",
72 [IB_EVENT_CLIENT_REREGISTER
] = "client reregister",
73 [IB_EVENT_GID_CHANGE
] = "GID changed",
76 const char *__attribute_const__
ib_event_msg(enum ib_event_type event
)
80 return (index
< ARRAY_SIZE(ib_events
) && ib_events
[index
]) ?
81 ib_events
[index
] : "unrecognized event";
83 EXPORT_SYMBOL(ib_event_msg
);
85 static const char * const wc_statuses
[] = {
86 [IB_WC_SUCCESS
] = "success",
87 [IB_WC_LOC_LEN_ERR
] = "local length error",
88 [IB_WC_LOC_QP_OP_ERR
] = "local QP operation error",
89 [IB_WC_LOC_EEC_OP_ERR
] = "local EE context operation error",
90 [IB_WC_LOC_PROT_ERR
] = "local protection error",
91 [IB_WC_WR_FLUSH_ERR
] = "WR flushed",
92 [IB_WC_MW_BIND_ERR
] = "memory management operation error",
93 [IB_WC_BAD_RESP_ERR
] = "bad response error",
94 [IB_WC_LOC_ACCESS_ERR
] = "local access error",
95 [IB_WC_REM_INV_REQ_ERR
] = "invalid request error",
96 [IB_WC_REM_ACCESS_ERR
] = "remote access error",
97 [IB_WC_REM_OP_ERR
] = "remote operation error",
98 [IB_WC_RETRY_EXC_ERR
] = "transport retry counter exceeded",
99 [IB_WC_RNR_RETRY_EXC_ERR
] = "RNR retry counter exceeded",
100 [IB_WC_LOC_RDD_VIOL_ERR
] = "local RDD violation error",
101 [IB_WC_REM_INV_RD_REQ_ERR
] = "remote invalid RD request",
102 [IB_WC_REM_ABORT_ERR
] = "operation aborted",
103 [IB_WC_INV_EECN_ERR
] = "invalid EE context number",
104 [IB_WC_INV_EEC_STATE_ERR
] = "invalid EE context state",
105 [IB_WC_FATAL_ERR
] = "fatal error",
106 [IB_WC_RESP_TIMEOUT_ERR
] = "response timeout error",
107 [IB_WC_GENERAL_ERR
] = "general error",
110 const char *__attribute_const__
ib_wc_status_msg(enum ib_wc_status status
)
112 size_t index
= status
;
114 return (index
< ARRAY_SIZE(wc_statuses
) && wc_statuses
[index
]) ?
115 wc_statuses
[index
] : "unrecognized status";
117 EXPORT_SYMBOL(ib_wc_status_msg
);
119 __attribute_const__
int ib_rate_to_mult(enum ib_rate rate
)
122 case IB_RATE_2_5_GBPS
: return 1;
123 case IB_RATE_5_GBPS
: return 2;
124 case IB_RATE_10_GBPS
: return 4;
125 case IB_RATE_20_GBPS
: return 8;
126 case IB_RATE_30_GBPS
: return 12;
127 case IB_RATE_40_GBPS
: return 16;
128 case IB_RATE_60_GBPS
: return 24;
129 case IB_RATE_80_GBPS
: return 32;
130 case IB_RATE_120_GBPS
: return 48;
134 EXPORT_SYMBOL(ib_rate_to_mult
);
136 __attribute_const__
enum ib_rate
mult_to_ib_rate(int mult
)
139 case 1: return IB_RATE_2_5_GBPS
;
140 case 2: return IB_RATE_5_GBPS
;
141 case 4: return IB_RATE_10_GBPS
;
142 case 8: return IB_RATE_20_GBPS
;
143 case 12: return IB_RATE_30_GBPS
;
144 case 16: return IB_RATE_40_GBPS
;
145 case 24: return IB_RATE_60_GBPS
;
146 case 32: return IB_RATE_80_GBPS
;
147 case 48: return IB_RATE_120_GBPS
;
148 default: return IB_RATE_PORT_CURRENT
;
151 EXPORT_SYMBOL(mult_to_ib_rate
);
153 __attribute_const__
int ib_rate_to_mbps(enum ib_rate rate
)
156 case IB_RATE_2_5_GBPS
: return 2500;
157 case IB_RATE_5_GBPS
: return 5000;
158 case IB_RATE_10_GBPS
: return 10000;
159 case IB_RATE_20_GBPS
: return 20000;
160 case IB_RATE_30_GBPS
: return 30000;
161 case IB_RATE_40_GBPS
: return 40000;
162 case IB_RATE_60_GBPS
: return 60000;
163 case IB_RATE_80_GBPS
: return 80000;
164 case IB_RATE_120_GBPS
: return 120000;
165 case IB_RATE_14_GBPS
: return 14062;
166 case IB_RATE_56_GBPS
: return 56250;
167 case IB_RATE_112_GBPS
: return 112500;
168 case IB_RATE_168_GBPS
: return 168750;
169 case IB_RATE_25_GBPS
: return 25781;
170 case IB_RATE_100_GBPS
: return 103125;
171 case IB_RATE_200_GBPS
: return 206250;
172 case IB_RATE_300_GBPS
: return 309375;
176 EXPORT_SYMBOL(ib_rate_to_mbps
);
178 __attribute_const__
enum rdma_transport_type
179 rdma_node_get_transport(enum rdma_node_type node_type
)
182 case RDMA_NODE_IB_CA
:
183 case RDMA_NODE_IB_SWITCH
:
184 case RDMA_NODE_IB_ROUTER
:
185 return RDMA_TRANSPORT_IB
;
187 return RDMA_TRANSPORT_IWARP
;
188 case RDMA_NODE_USNIC
:
189 return RDMA_TRANSPORT_USNIC
;
190 case RDMA_NODE_USNIC_UDP
:
191 return RDMA_TRANSPORT_USNIC_UDP
;
197 EXPORT_SYMBOL(rdma_node_get_transport
);
199 enum rdma_link_layer
rdma_port_get_link_layer(struct ib_device
*device
, u8 port_num
)
201 if (device
->get_link_layer
)
202 return device
->get_link_layer(device
, port_num
);
204 switch (rdma_node_get_transport(device
->node_type
)) {
205 case RDMA_TRANSPORT_IB
:
206 return IB_LINK_LAYER_INFINIBAND
;
207 case RDMA_TRANSPORT_IWARP
:
208 case RDMA_TRANSPORT_USNIC
:
209 case RDMA_TRANSPORT_USNIC_UDP
:
210 return IB_LINK_LAYER_ETHERNET
;
212 return IB_LINK_LAYER_UNSPECIFIED
;
215 EXPORT_SYMBOL(rdma_port_get_link_layer
);
217 /* Protection domains */
220 * ib_alloc_pd - Allocates an unused protection domain.
221 * @device: The device on which to allocate the protection domain.
223 * A protection domain object provides an association between QPs, shared
224 * receive queues, address handles, memory regions, and memory windows.
226 * Every PD has a local_dma_lkey which can be used as the lkey value for local
229 struct ib_pd
*ib_alloc_pd(struct ib_device
*device
)
233 pd
= device
->alloc_pd(device
, NULL
, NULL
);
240 atomic_set(&pd
->usecnt
, 0);
242 if (device
->attrs
.device_cap_flags
& IB_DEVICE_LOCAL_DMA_LKEY
)
243 pd
->local_dma_lkey
= device
->local_dma_lkey
;
247 mr
= ib_get_dma_mr(pd
, IB_ACCESS_LOCAL_WRITE
);
250 return (struct ib_pd
*)mr
;
254 pd
->local_dma_lkey
= pd
->local_mr
->lkey
;
258 EXPORT_SYMBOL(ib_alloc_pd
);
261 * ib_dealloc_pd - Deallocates a protection domain.
262 * @pd: The protection domain to deallocate.
264 * It is an error to call this function while any resources in the pd still
265 * exist. The caller is responsible to synchronously destroy them and
266 * guarantee no new allocations will happen.
268 void ib_dealloc_pd(struct ib_pd
*pd
)
273 ret
= ib_dereg_mr(pd
->local_mr
);
278 /* uverbs manipulates usecnt with proper locking, while the kabi
279 requires the caller to guarantee we can't race here. */
280 WARN_ON(atomic_read(&pd
->usecnt
));
282 /* Making delalloc_pd a void return is a WIP, no driver should return
284 ret
= pd
->device
->dealloc_pd(pd
);
285 WARN_ONCE(ret
, "Infiniband HW driver failed dealloc_pd");
287 EXPORT_SYMBOL(ib_dealloc_pd
);
289 /* Address handles */
291 struct ib_ah
*ib_create_ah(struct ib_pd
*pd
, struct ib_ah_attr
*ah_attr
)
295 ah
= pd
->device
->create_ah(pd
, ah_attr
);
298 ah
->device
= pd
->device
;
301 atomic_inc(&pd
->usecnt
);
306 EXPORT_SYMBOL(ib_create_ah
);
308 static int ib_get_header_version(const union rdma_network_hdr
*hdr
)
310 const struct iphdr
*ip4h
= (struct iphdr
*)&hdr
->roce4grh
;
311 struct iphdr ip4h_checked
;
312 const struct ipv6hdr
*ip6h
= (struct ipv6hdr
*)&hdr
->ibgrh
;
314 /* If it's IPv6, the version must be 6, otherwise, the first
315 * 20 bytes (before the IPv4 header) are garbled.
317 if (ip6h
->version
!= 6)
318 return (ip4h
->version
== 4) ? 4 : 0;
319 /* version may be 6 or 4 because the first 20 bytes could be garbled */
321 /* RoCE v2 requires no options, thus header length
328 * We can't write on scattered buffers so we need to copy to
331 memcpy(&ip4h_checked
, ip4h
, sizeof(ip4h_checked
));
332 ip4h_checked
.check
= 0;
333 ip4h_checked
.check
= ip_fast_csum((u8
*)&ip4h_checked
, 5);
334 /* if IPv4 header checksum is OK, believe it */
335 if (ip4h
->check
== ip4h_checked
.check
)
340 static enum rdma_network_type
ib_get_net_type_by_grh(struct ib_device
*device
,
342 const struct ib_grh
*grh
)
346 if (rdma_protocol_ib(device
, port_num
))
347 return RDMA_NETWORK_IB
;
349 grh_version
= ib_get_header_version((union rdma_network_hdr
*)grh
);
351 if (grh_version
== 4)
352 return RDMA_NETWORK_IPV4
;
354 if (grh
->next_hdr
== IPPROTO_UDP
)
355 return RDMA_NETWORK_IPV6
;
357 return RDMA_NETWORK_ROCE_V1
;
360 struct find_gid_index_context
{
362 enum ib_gid_type gid_type
;
365 static bool find_gid_index(const union ib_gid
*gid
,
366 const struct ib_gid_attr
*gid_attr
,
369 struct find_gid_index_context
*ctx
=
370 (struct find_gid_index_context
*)context
;
372 if (ctx
->gid_type
!= gid_attr
->gid_type
)
375 if ((!!(ctx
->vlan_id
!= 0xffff) == !is_vlan_dev(gid_attr
->ndev
)) ||
376 (is_vlan_dev(gid_attr
->ndev
) &&
377 vlan_dev_vlan_id(gid_attr
->ndev
) != ctx
->vlan_id
))
383 static int get_sgid_index_from_eth(struct ib_device
*device
, u8 port_num
,
384 u16 vlan_id
, const union ib_gid
*sgid
,
385 enum ib_gid_type gid_type
,
388 struct find_gid_index_context context
= {.vlan_id
= vlan_id
,
389 .gid_type
= gid_type
};
391 return ib_find_gid_by_filter(device
, sgid
, port_num
, find_gid_index
,
392 &context
, gid_index
);
395 static int get_gids_from_rdma_hdr(union rdma_network_hdr
*hdr
,
396 enum rdma_network_type net_type
,
397 union ib_gid
*sgid
, union ib_gid
*dgid
)
399 struct sockaddr_in src_in
;
400 struct sockaddr_in dst_in
;
401 __be32 src_saddr
, dst_saddr
;
406 if (net_type
== RDMA_NETWORK_IPV4
) {
407 memcpy(&src_in
.sin_addr
.s_addr
,
408 &hdr
->roce4grh
.saddr
, 4);
409 memcpy(&dst_in
.sin_addr
.s_addr
,
410 &hdr
->roce4grh
.daddr
, 4);
411 src_saddr
= src_in
.sin_addr
.s_addr
;
412 dst_saddr
= dst_in
.sin_addr
.s_addr
;
413 ipv6_addr_set_v4mapped(src_saddr
,
414 (struct in6_addr
*)sgid
);
415 ipv6_addr_set_v4mapped(dst_saddr
,
416 (struct in6_addr
*)dgid
);
418 } else if (net_type
== RDMA_NETWORK_IPV6
||
419 net_type
== RDMA_NETWORK_IB
) {
420 *dgid
= hdr
->ibgrh
.dgid
;
421 *sgid
= hdr
->ibgrh
.sgid
;
428 int ib_init_ah_from_wc(struct ib_device
*device
, u8 port_num
,
429 const struct ib_wc
*wc
, const struct ib_grh
*grh
,
430 struct ib_ah_attr
*ah_attr
)
435 enum rdma_network_type net_type
= RDMA_NETWORK_IB
;
436 enum ib_gid_type gid_type
= IB_GID_TYPE_IB
;
441 memset(ah_attr
, 0, sizeof *ah_attr
);
442 if (rdma_cap_eth_ah(device
, port_num
)) {
443 if (wc
->wc_flags
& IB_WC_WITH_NETWORK_HDR_TYPE
)
444 net_type
= wc
->network_hdr_type
;
446 net_type
= ib_get_net_type_by_grh(device
, port_num
, grh
);
447 gid_type
= ib_network_to_gid_type(net_type
);
449 ret
= get_gids_from_rdma_hdr((union rdma_network_hdr
*)grh
, net_type
,
454 if (rdma_protocol_roce(device
, port_num
)) {
456 u16 vlan_id
= wc
->wc_flags
& IB_WC_WITH_VLAN
?
457 wc
->vlan_id
: 0xffff;
458 struct net_device
*idev
;
459 struct net_device
*resolved_dev
;
461 if (!(wc
->wc_flags
& IB_WC_GRH
))
464 if (!device
->get_netdev
)
467 idev
= device
->get_netdev(device
, port_num
);
471 ret
= rdma_addr_find_l2_eth_by_grh(&dgid
, &sgid
,
473 wc
->wc_flags
& IB_WC_WITH_VLAN
?
475 &if_index
, &hoplimit
);
481 resolved_dev
= dev_get_by_index(&init_net
, if_index
);
482 if (resolved_dev
->flags
& IFF_LOOPBACK
) {
483 dev_put(resolved_dev
);
485 dev_hold(resolved_dev
);
488 if (resolved_dev
!= idev
&& !rdma_is_upper_dev_rcu(idev
,
493 dev_put(resolved_dev
);
497 ret
= get_sgid_index_from_eth(device
, port_num
, vlan_id
,
498 &dgid
, gid_type
, &gid_index
);
503 ah_attr
->dlid
= wc
->slid
;
504 ah_attr
->sl
= wc
->sl
;
505 ah_attr
->src_path_bits
= wc
->dlid_path_bits
;
506 ah_attr
->port_num
= port_num
;
508 if (wc
->wc_flags
& IB_WC_GRH
) {
509 ah_attr
->ah_flags
= IB_AH_GRH
;
510 ah_attr
->grh
.dgid
= sgid
;
512 if (!rdma_cap_eth_ah(device
, port_num
)) {
513 ret
= ib_find_cached_gid_by_port(device
, &dgid
,
521 ah_attr
->grh
.sgid_index
= (u8
) gid_index
;
522 flow_class
= be32_to_cpu(grh
->version_tclass_flow
);
523 ah_attr
->grh
.flow_label
= flow_class
& 0xFFFFF;
524 ah_attr
->grh
.hop_limit
= hoplimit
;
525 ah_attr
->grh
.traffic_class
= (flow_class
>> 20) & 0xFF;
529 EXPORT_SYMBOL(ib_init_ah_from_wc
);
531 struct ib_ah
*ib_create_ah_from_wc(struct ib_pd
*pd
, const struct ib_wc
*wc
,
532 const struct ib_grh
*grh
, u8 port_num
)
534 struct ib_ah_attr ah_attr
;
537 ret
= ib_init_ah_from_wc(pd
->device
, port_num
, wc
, grh
, &ah_attr
);
541 return ib_create_ah(pd
, &ah_attr
);
543 EXPORT_SYMBOL(ib_create_ah_from_wc
);
545 int ib_modify_ah(struct ib_ah
*ah
, struct ib_ah_attr
*ah_attr
)
547 return ah
->device
->modify_ah
?
548 ah
->device
->modify_ah(ah
, ah_attr
) :
551 EXPORT_SYMBOL(ib_modify_ah
);
553 int ib_query_ah(struct ib_ah
*ah
, struct ib_ah_attr
*ah_attr
)
555 return ah
->device
->query_ah
?
556 ah
->device
->query_ah(ah
, ah_attr
) :
559 EXPORT_SYMBOL(ib_query_ah
);
561 int ib_destroy_ah(struct ib_ah
*ah
)
567 ret
= ah
->device
->destroy_ah(ah
);
569 atomic_dec(&pd
->usecnt
);
573 EXPORT_SYMBOL(ib_destroy_ah
);
575 /* Shared receive queues */
577 struct ib_srq
*ib_create_srq(struct ib_pd
*pd
,
578 struct ib_srq_init_attr
*srq_init_attr
)
582 if (!pd
->device
->create_srq
)
583 return ERR_PTR(-ENOSYS
);
585 srq
= pd
->device
->create_srq(pd
, srq_init_attr
, NULL
);
588 srq
->device
= pd
->device
;
591 srq
->event_handler
= srq_init_attr
->event_handler
;
592 srq
->srq_context
= srq_init_attr
->srq_context
;
593 srq
->srq_type
= srq_init_attr
->srq_type
;
594 if (srq
->srq_type
== IB_SRQT_XRC
) {
595 srq
->ext
.xrc
.xrcd
= srq_init_attr
->ext
.xrc
.xrcd
;
596 srq
->ext
.xrc
.cq
= srq_init_attr
->ext
.xrc
.cq
;
597 atomic_inc(&srq
->ext
.xrc
.xrcd
->usecnt
);
598 atomic_inc(&srq
->ext
.xrc
.cq
->usecnt
);
600 atomic_inc(&pd
->usecnt
);
601 atomic_set(&srq
->usecnt
, 0);
606 EXPORT_SYMBOL(ib_create_srq
);
608 int ib_modify_srq(struct ib_srq
*srq
,
609 struct ib_srq_attr
*srq_attr
,
610 enum ib_srq_attr_mask srq_attr_mask
)
612 return srq
->device
->modify_srq
?
613 srq
->device
->modify_srq(srq
, srq_attr
, srq_attr_mask
, NULL
) :
616 EXPORT_SYMBOL(ib_modify_srq
);
618 int ib_query_srq(struct ib_srq
*srq
,
619 struct ib_srq_attr
*srq_attr
)
621 return srq
->device
->query_srq
?
622 srq
->device
->query_srq(srq
, srq_attr
) : -ENOSYS
;
624 EXPORT_SYMBOL(ib_query_srq
);
626 int ib_destroy_srq(struct ib_srq
*srq
)
629 enum ib_srq_type srq_type
;
630 struct ib_xrcd
*uninitialized_var(xrcd
);
631 struct ib_cq
*uninitialized_var(cq
);
634 if (atomic_read(&srq
->usecnt
))
638 srq_type
= srq
->srq_type
;
639 if (srq_type
== IB_SRQT_XRC
) {
640 xrcd
= srq
->ext
.xrc
.xrcd
;
641 cq
= srq
->ext
.xrc
.cq
;
644 ret
= srq
->device
->destroy_srq(srq
);
646 atomic_dec(&pd
->usecnt
);
647 if (srq_type
== IB_SRQT_XRC
) {
648 atomic_dec(&xrcd
->usecnt
);
649 atomic_dec(&cq
->usecnt
);
655 EXPORT_SYMBOL(ib_destroy_srq
);
659 static void __ib_shared_qp_event_handler(struct ib_event
*event
, void *context
)
661 struct ib_qp
*qp
= context
;
664 spin_lock_irqsave(&qp
->device
->event_handler_lock
, flags
);
665 list_for_each_entry(event
->element
.qp
, &qp
->open_list
, open_list
)
666 if (event
->element
.qp
->event_handler
)
667 event
->element
.qp
->event_handler(event
, event
->element
.qp
->qp_context
);
668 spin_unlock_irqrestore(&qp
->device
->event_handler_lock
, flags
);
671 static void __ib_insert_xrcd_qp(struct ib_xrcd
*xrcd
, struct ib_qp
*qp
)
673 mutex_lock(&xrcd
->tgt_qp_mutex
);
674 list_add(&qp
->xrcd_list
, &xrcd
->tgt_qp_list
);
675 mutex_unlock(&xrcd
->tgt_qp_mutex
);
678 static struct ib_qp
*__ib_open_qp(struct ib_qp
*real_qp
,
679 void (*event_handler
)(struct ib_event
*, void *),
685 qp
= kzalloc(sizeof *qp
, GFP_KERNEL
);
687 return ERR_PTR(-ENOMEM
);
689 qp
->real_qp
= real_qp
;
690 atomic_inc(&real_qp
->usecnt
);
691 qp
->device
= real_qp
->device
;
692 qp
->event_handler
= event_handler
;
693 qp
->qp_context
= qp_context
;
694 qp
->qp_num
= real_qp
->qp_num
;
695 qp
->qp_type
= real_qp
->qp_type
;
697 spin_lock_irqsave(&real_qp
->device
->event_handler_lock
, flags
);
698 list_add(&qp
->open_list
, &real_qp
->open_list
);
699 spin_unlock_irqrestore(&real_qp
->device
->event_handler_lock
, flags
);
704 struct ib_qp
*ib_open_qp(struct ib_xrcd
*xrcd
,
705 struct ib_qp_open_attr
*qp_open_attr
)
707 struct ib_qp
*qp
, *real_qp
;
709 if (qp_open_attr
->qp_type
!= IB_QPT_XRC_TGT
)
710 return ERR_PTR(-EINVAL
);
712 qp
= ERR_PTR(-EINVAL
);
713 mutex_lock(&xrcd
->tgt_qp_mutex
);
714 list_for_each_entry(real_qp
, &xrcd
->tgt_qp_list
, xrcd_list
) {
715 if (real_qp
->qp_num
== qp_open_attr
->qp_num
) {
716 qp
= __ib_open_qp(real_qp
, qp_open_attr
->event_handler
,
717 qp_open_attr
->qp_context
);
721 mutex_unlock(&xrcd
->tgt_qp_mutex
);
724 EXPORT_SYMBOL(ib_open_qp
);
726 static struct ib_qp
*ib_create_xrc_qp(struct ib_qp
*qp
,
727 struct ib_qp_init_attr
*qp_init_attr
)
729 struct ib_qp
*real_qp
= qp
;
731 qp
->event_handler
= __ib_shared_qp_event_handler
;
734 qp
->send_cq
= qp
->recv_cq
= NULL
;
736 qp
->xrcd
= qp_init_attr
->xrcd
;
737 atomic_inc(&qp_init_attr
->xrcd
->usecnt
);
738 INIT_LIST_HEAD(&qp
->open_list
);
740 qp
= __ib_open_qp(real_qp
, qp_init_attr
->event_handler
,
741 qp_init_attr
->qp_context
);
743 __ib_insert_xrcd_qp(qp_init_attr
->xrcd
, real_qp
);
745 real_qp
->device
->destroy_qp(real_qp
);
749 struct ib_qp
*ib_create_qp(struct ib_pd
*pd
,
750 struct ib_qp_init_attr
*qp_init_attr
)
752 struct ib_device
*device
= pd
? pd
->device
: qp_init_attr
->xrcd
->device
;
755 qp
= device
->create_qp(pd
, qp_init_attr
, NULL
);
762 qp
->qp_type
= qp_init_attr
->qp_type
;
764 atomic_set(&qp
->usecnt
, 0);
766 spin_lock_init(&qp
->mr_lock
);
768 if (qp_init_attr
->qp_type
== IB_QPT_XRC_TGT
)
769 return ib_create_xrc_qp(qp
, qp_init_attr
);
771 qp
->event_handler
= qp_init_attr
->event_handler
;
772 qp
->qp_context
= qp_init_attr
->qp_context
;
773 if (qp_init_attr
->qp_type
== IB_QPT_XRC_INI
) {
777 qp
->recv_cq
= qp_init_attr
->recv_cq
;
778 atomic_inc(&qp_init_attr
->recv_cq
->usecnt
);
779 qp
->srq
= qp_init_attr
->srq
;
781 atomic_inc(&qp_init_attr
->srq
->usecnt
);
785 qp
->send_cq
= qp_init_attr
->send_cq
;
788 atomic_inc(&pd
->usecnt
);
789 atomic_inc(&qp_init_attr
->send_cq
->usecnt
);
792 EXPORT_SYMBOL(ib_create_qp
);
794 static const struct {
796 enum ib_qp_attr_mask req_param
[IB_QPT_MAX
];
797 enum ib_qp_attr_mask opt_param
[IB_QPT_MAX
];
798 } qp_state_table
[IB_QPS_ERR
+ 1][IB_QPS_ERR
+ 1] = {
800 [IB_QPS_RESET
] = { .valid
= 1 },
804 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
807 [IB_QPT_RAW_PACKET
] = IB_QP_PORT
,
808 [IB_QPT_UC
] = (IB_QP_PKEY_INDEX
|
811 [IB_QPT_RC
] = (IB_QP_PKEY_INDEX
|
814 [IB_QPT_XRC_INI
] = (IB_QP_PKEY_INDEX
|
817 [IB_QPT_XRC_TGT
] = (IB_QP_PKEY_INDEX
|
820 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
822 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
828 [IB_QPS_RESET
] = { .valid
= 1 },
829 [IB_QPS_ERR
] = { .valid
= 1 },
833 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
836 [IB_QPT_UC
] = (IB_QP_PKEY_INDEX
|
839 [IB_QPT_RC
] = (IB_QP_PKEY_INDEX
|
842 [IB_QPT_XRC_INI
] = (IB_QP_PKEY_INDEX
|
845 [IB_QPT_XRC_TGT
] = (IB_QP_PKEY_INDEX
|
848 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
850 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
857 [IB_QPT_UC
] = (IB_QP_AV
|
861 [IB_QPT_RC
] = (IB_QP_AV
|
865 IB_QP_MAX_DEST_RD_ATOMIC
|
866 IB_QP_MIN_RNR_TIMER
),
867 [IB_QPT_XRC_INI
] = (IB_QP_AV
|
871 [IB_QPT_XRC_TGT
] = (IB_QP_AV
|
875 IB_QP_MAX_DEST_RD_ATOMIC
|
876 IB_QP_MIN_RNR_TIMER
),
879 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
881 [IB_QPT_UC
] = (IB_QP_ALT_PATH
|
884 [IB_QPT_RC
] = (IB_QP_ALT_PATH
|
887 [IB_QPT_XRC_INI
] = (IB_QP_ALT_PATH
|
890 [IB_QPT_XRC_TGT
] = (IB_QP_ALT_PATH
|
893 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
895 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
901 [IB_QPS_RESET
] = { .valid
= 1 },
902 [IB_QPS_ERR
] = { .valid
= 1 },
906 [IB_QPT_UD
] = IB_QP_SQ_PSN
,
907 [IB_QPT_UC
] = IB_QP_SQ_PSN
,
908 [IB_QPT_RC
] = (IB_QP_TIMEOUT
|
912 IB_QP_MAX_QP_RD_ATOMIC
),
913 [IB_QPT_XRC_INI
] = (IB_QP_TIMEOUT
|
917 IB_QP_MAX_QP_RD_ATOMIC
),
918 [IB_QPT_XRC_TGT
] = (IB_QP_TIMEOUT
|
920 [IB_QPT_SMI
] = IB_QP_SQ_PSN
,
921 [IB_QPT_GSI
] = IB_QP_SQ_PSN
,
924 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
926 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
929 IB_QP_PATH_MIG_STATE
),
930 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
933 IB_QP_MIN_RNR_TIMER
|
934 IB_QP_PATH_MIG_STATE
),
935 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
938 IB_QP_PATH_MIG_STATE
),
939 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
942 IB_QP_MIN_RNR_TIMER
|
943 IB_QP_PATH_MIG_STATE
),
944 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
946 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
952 [IB_QPS_RESET
] = { .valid
= 1 },
953 [IB_QPS_ERR
] = { .valid
= 1 },
957 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
959 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
962 IB_QP_PATH_MIG_STATE
),
963 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
966 IB_QP_PATH_MIG_STATE
|
967 IB_QP_MIN_RNR_TIMER
),
968 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
971 IB_QP_PATH_MIG_STATE
),
972 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
975 IB_QP_PATH_MIG_STATE
|
976 IB_QP_MIN_RNR_TIMER
),
977 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
979 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
986 [IB_QPT_UD
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
987 [IB_QPT_UC
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
988 [IB_QPT_RC
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
989 [IB_QPT_XRC_INI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
990 [IB_QPT_XRC_TGT
] = IB_QP_EN_SQD_ASYNC_NOTIFY
, /* ??? */
991 [IB_QPT_SMI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
,
992 [IB_QPT_GSI
] = IB_QP_EN_SQD_ASYNC_NOTIFY
997 [IB_QPS_RESET
] = { .valid
= 1 },
998 [IB_QPS_ERR
] = { .valid
= 1 },
1002 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
1004 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
1006 IB_QP_ACCESS_FLAGS
|
1007 IB_QP_PATH_MIG_STATE
),
1008 [IB_QPT_RC
] = (IB_QP_CUR_STATE
|
1010 IB_QP_ACCESS_FLAGS
|
1011 IB_QP_MIN_RNR_TIMER
|
1012 IB_QP_PATH_MIG_STATE
),
1013 [IB_QPT_XRC_INI
] = (IB_QP_CUR_STATE
|
1015 IB_QP_ACCESS_FLAGS
|
1016 IB_QP_PATH_MIG_STATE
),
1017 [IB_QPT_XRC_TGT
] = (IB_QP_CUR_STATE
|
1019 IB_QP_ACCESS_FLAGS
|
1020 IB_QP_MIN_RNR_TIMER
|
1021 IB_QP_PATH_MIG_STATE
),
1022 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
1024 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
1031 [IB_QPT_UD
] = (IB_QP_PKEY_INDEX
|
1033 [IB_QPT_UC
] = (IB_QP_AV
|
1035 IB_QP_ACCESS_FLAGS
|
1037 IB_QP_PATH_MIG_STATE
),
1038 [IB_QPT_RC
] = (IB_QP_PORT
|
1043 IB_QP_MAX_QP_RD_ATOMIC
|
1044 IB_QP_MAX_DEST_RD_ATOMIC
|
1046 IB_QP_ACCESS_FLAGS
|
1048 IB_QP_MIN_RNR_TIMER
|
1049 IB_QP_PATH_MIG_STATE
),
1050 [IB_QPT_XRC_INI
] = (IB_QP_PORT
|
1055 IB_QP_MAX_QP_RD_ATOMIC
|
1057 IB_QP_ACCESS_FLAGS
|
1059 IB_QP_PATH_MIG_STATE
),
1060 [IB_QPT_XRC_TGT
] = (IB_QP_PORT
|
1063 IB_QP_MAX_DEST_RD_ATOMIC
|
1065 IB_QP_ACCESS_FLAGS
|
1067 IB_QP_MIN_RNR_TIMER
|
1068 IB_QP_PATH_MIG_STATE
),
1069 [IB_QPT_SMI
] = (IB_QP_PKEY_INDEX
|
1071 [IB_QPT_GSI
] = (IB_QP_PKEY_INDEX
|
1077 [IB_QPS_RESET
] = { .valid
= 1 },
1078 [IB_QPS_ERR
] = { .valid
= 1 },
1082 [IB_QPT_UD
] = (IB_QP_CUR_STATE
|
1084 [IB_QPT_UC
] = (IB_QP_CUR_STATE
|
1085 IB_QP_ACCESS_FLAGS
),
1086 [IB_QPT_SMI
] = (IB_QP_CUR_STATE
|
1088 [IB_QPT_GSI
] = (IB_QP_CUR_STATE
|
1094 [IB_QPS_RESET
] = { .valid
= 1 },
1095 [IB_QPS_ERR
] = { .valid
= 1 }
1099 int ib_modify_qp_is_ok(enum ib_qp_state cur_state
, enum ib_qp_state next_state
,
1100 enum ib_qp_type type
, enum ib_qp_attr_mask mask
,
1101 enum rdma_link_layer ll
)
1103 enum ib_qp_attr_mask req_param
, opt_param
;
1105 if (cur_state
< 0 || cur_state
> IB_QPS_ERR
||
1106 next_state
< 0 || next_state
> IB_QPS_ERR
)
1109 if (mask
& IB_QP_CUR_STATE
&&
1110 cur_state
!= IB_QPS_RTR
&& cur_state
!= IB_QPS_RTS
&&
1111 cur_state
!= IB_QPS_SQD
&& cur_state
!= IB_QPS_SQE
)
1114 if (!qp_state_table
[cur_state
][next_state
].valid
)
1117 req_param
= qp_state_table
[cur_state
][next_state
].req_param
[type
];
1118 opt_param
= qp_state_table
[cur_state
][next_state
].opt_param
[type
];
1120 if ((mask
& req_param
) != req_param
)
1123 if (mask
& ~(req_param
| opt_param
| IB_QP_STATE
))
1128 EXPORT_SYMBOL(ib_modify_qp_is_ok
);
1130 int ib_resolve_eth_dmac(struct ib_qp
*qp
,
1131 struct ib_qp_attr
*qp_attr
, int *qp_attr_mask
)
1135 if (*qp_attr_mask
& IB_QP_AV
) {
1136 if (qp_attr
->ah_attr
.port_num
< rdma_start_port(qp
->device
) ||
1137 qp_attr
->ah_attr
.port_num
> rdma_end_port(qp
->device
))
1140 if (!rdma_cap_eth_ah(qp
->device
, qp_attr
->ah_attr
.port_num
))
1143 if (rdma_link_local_addr((struct in6_addr
*)qp_attr
->ah_attr
.grh
.dgid
.raw
)) {
1144 rdma_get_ll_mac((struct in6_addr
*)qp_attr
->ah_attr
.grh
.dgid
.raw
,
1145 qp_attr
->ah_attr
.dmac
);
1148 struct ib_gid_attr sgid_attr
;
1152 ret
= ib_query_gid(qp
->device
,
1153 qp_attr
->ah_attr
.port_num
,
1154 qp_attr
->ah_attr
.grh
.sgid_index
,
1157 if (ret
|| !sgid_attr
.ndev
) {
1163 ifindex
= sgid_attr
.ndev
->ifindex
;
1165 ret
= rdma_addr_find_l2_eth_by_grh(&sgid
,
1166 &qp_attr
->ah_attr
.grh
.dgid
,
1167 qp_attr
->ah_attr
.dmac
,
1168 NULL
, &ifindex
, &hop_limit
);
1170 dev_put(sgid_attr
.ndev
);
1172 qp_attr
->ah_attr
.grh
.hop_limit
= hop_limit
;
1178 EXPORT_SYMBOL(ib_resolve_eth_dmac
);
1181 int ib_modify_qp(struct ib_qp
*qp
,
1182 struct ib_qp_attr
*qp_attr
,
1187 ret
= ib_resolve_eth_dmac(qp
, qp_attr
, &qp_attr_mask
);
1191 return qp
->device
->modify_qp(qp
->real_qp
, qp_attr
, qp_attr_mask
, NULL
);
1193 EXPORT_SYMBOL(ib_modify_qp
);
1195 int ib_query_qp(struct ib_qp
*qp
,
1196 struct ib_qp_attr
*qp_attr
,
1198 struct ib_qp_init_attr
*qp_init_attr
)
1200 return qp
->device
->query_qp
?
1201 qp
->device
->query_qp(qp
->real_qp
, qp_attr
, qp_attr_mask
, qp_init_attr
) :
1204 EXPORT_SYMBOL(ib_query_qp
);
1206 int ib_close_qp(struct ib_qp
*qp
)
1208 struct ib_qp
*real_qp
;
1209 unsigned long flags
;
1211 real_qp
= qp
->real_qp
;
1215 spin_lock_irqsave(&real_qp
->device
->event_handler_lock
, flags
);
1216 list_del(&qp
->open_list
);
1217 spin_unlock_irqrestore(&real_qp
->device
->event_handler_lock
, flags
);
1219 atomic_dec(&real_qp
->usecnt
);
1224 EXPORT_SYMBOL(ib_close_qp
);
1226 static int __ib_destroy_shared_qp(struct ib_qp
*qp
)
1228 struct ib_xrcd
*xrcd
;
1229 struct ib_qp
*real_qp
;
1232 real_qp
= qp
->real_qp
;
1233 xrcd
= real_qp
->xrcd
;
1235 mutex_lock(&xrcd
->tgt_qp_mutex
);
1237 if (atomic_read(&real_qp
->usecnt
) == 0)
1238 list_del(&real_qp
->xrcd_list
);
1241 mutex_unlock(&xrcd
->tgt_qp_mutex
);
1244 ret
= ib_destroy_qp(real_qp
);
1246 atomic_dec(&xrcd
->usecnt
);
1248 __ib_insert_xrcd_qp(xrcd
, real_qp
);
1254 int ib_destroy_qp(struct ib_qp
*qp
)
1257 struct ib_cq
*scq
, *rcq
;
1261 WARN_ON_ONCE(qp
->mrs_used
> 0);
1263 if (atomic_read(&qp
->usecnt
))
1266 if (qp
->real_qp
!= qp
)
1267 return __ib_destroy_shared_qp(qp
);
1274 ret
= qp
->device
->destroy_qp(qp
);
1277 atomic_dec(&pd
->usecnt
);
1279 atomic_dec(&scq
->usecnt
);
1281 atomic_dec(&rcq
->usecnt
);
1283 atomic_dec(&srq
->usecnt
);
1288 EXPORT_SYMBOL(ib_destroy_qp
);
1290 /* Completion queues */
1292 struct ib_cq
*ib_create_cq(struct ib_device
*device
,
1293 ib_comp_handler comp_handler
,
1294 void (*event_handler
)(struct ib_event
*, void *),
1296 const struct ib_cq_init_attr
*cq_attr
)
1300 cq
= device
->create_cq(device
, cq_attr
, NULL
, NULL
);
1303 cq
->device
= device
;
1305 cq
->comp_handler
= comp_handler
;
1306 cq
->event_handler
= event_handler
;
1307 cq
->cq_context
= cq_context
;
1308 atomic_set(&cq
->usecnt
, 0);
1313 EXPORT_SYMBOL(ib_create_cq
);
1315 int ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
)
1317 return cq
->device
->modify_cq
?
1318 cq
->device
->modify_cq(cq
, cq_count
, cq_period
) : -ENOSYS
;
1320 EXPORT_SYMBOL(ib_modify_cq
);
1322 int ib_destroy_cq(struct ib_cq
*cq
)
1324 if (atomic_read(&cq
->usecnt
))
1327 return cq
->device
->destroy_cq(cq
);
1329 EXPORT_SYMBOL(ib_destroy_cq
);
1331 int ib_resize_cq(struct ib_cq
*cq
, int cqe
)
1333 return cq
->device
->resize_cq
?
1334 cq
->device
->resize_cq(cq
, cqe
, NULL
) : -ENOSYS
;
1336 EXPORT_SYMBOL(ib_resize_cq
);
1338 /* Memory regions */
1340 struct ib_mr
*ib_get_dma_mr(struct ib_pd
*pd
, int mr_access_flags
)
1345 err
= ib_check_mr_access(mr_access_flags
);
1347 return ERR_PTR(err
);
1349 mr
= pd
->device
->get_dma_mr(pd
, mr_access_flags
);
1352 mr
->device
= pd
->device
;
1355 atomic_inc(&pd
->usecnt
);
1360 EXPORT_SYMBOL(ib_get_dma_mr
);
1362 int ib_dereg_mr(struct ib_mr
*mr
)
1364 struct ib_pd
*pd
= mr
->pd
;
1367 ret
= mr
->device
->dereg_mr(mr
);
1369 atomic_dec(&pd
->usecnt
);
1373 EXPORT_SYMBOL(ib_dereg_mr
);
1376 * ib_alloc_mr() - Allocates a memory region
1377 * @pd: protection domain associated with the region
1378 * @mr_type: memory region type
1379 * @max_num_sg: maximum sg entries available for registration.
1382 * Memory registeration page/sg lists must not exceed max_num_sg.
1383 * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
1384 * max_num_sg * used_page_size.
1387 struct ib_mr
*ib_alloc_mr(struct ib_pd
*pd
,
1388 enum ib_mr_type mr_type
,
1393 if (!pd
->device
->alloc_mr
)
1394 return ERR_PTR(-ENOSYS
);
1396 mr
= pd
->device
->alloc_mr(pd
, mr_type
, max_num_sg
);
1398 mr
->device
= pd
->device
;
1401 atomic_inc(&pd
->usecnt
);
1406 EXPORT_SYMBOL(ib_alloc_mr
);
1408 /* "Fast" memory regions */
1410 struct ib_fmr
*ib_alloc_fmr(struct ib_pd
*pd
,
1411 int mr_access_flags
,
1412 struct ib_fmr_attr
*fmr_attr
)
1416 if (!pd
->device
->alloc_fmr
)
1417 return ERR_PTR(-ENOSYS
);
1419 fmr
= pd
->device
->alloc_fmr(pd
, mr_access_flags
, fmr_attr
);
1421 fmr
->device
= pd
->device
;
1423 atomic_inc(&pd
->usecnt
);
1428 EXPORT_SYMBOL(ib_alloc_fmr
);
1430 int ib_unmap_fmr(struct list_head
*fmr_list
)
1434 if (list_empty(fmr_list
))
1437 fmr
= list_entry(fmr_list
->next
, struct ib_fmr
, list
);
1438 return fmr
->device
->unmap_fmr(fmr_list
);
1440 EXPORT_SYMBOL(ib_unmap_fmr
);
1442 int ib_dealloc_fmr(struct ib_fmr
*fmr
)
1448 ret
= fmr
->device
->dealloc_fmr(fmr
);
1450 atomic_dec(&pd
->usecnt
);
1454 EXPORT_SYMBOL(ib_dealloc_fmr
);
1456 /* Multicast groups */
1458 int ib_attach_mcast(struct ib_qp
*qp
, union ib_gid
*gid
, u16 lid
)
1462 if (!qp
->device
->attach_mcast
)
1464 if (gid
->raw
[0] != 0xff || qp
->qp_type
!= IB_QPT_UD
)
1467 ret
= qp
->device
->attach_mcast(qp
, gid
, lid
);
1469 atomic_inc(&qp
->usecnt
);
1472 EXPORT_SYMBOL(ib_attach_mcast
);
1474 int ib_detach_mcast(struct ib_qp
*qp
, union ib_gid
*gid
, u16 lid
)
1478 if (!qp
->device
->detach_mcast
)
1480 if (gid
->raw
[0] != 0xff || qp
->qp_type
!= IB_QPT_UD
)
1483 ret
= qp
->device
->detach_mcast(qp
, gid
, lid
);
1485 atomic_dec(&qp
->usecnt
);
1488 EXPORT_SYMBOL(ib_detach_mcast
);
1490 struct ib_xrcd
*ib_alloc_xrcd(struct ib_device
*device
)
1492 struct ib_xrcd
*xrcd
;
1494 if (!device
->alloc_xrcd
)
1495 return ERR_PTR(-ENOSYS
);
1497 xrcd
= device
->alloc_xrcd(device
, NULL
, NULL
);
1498 if (!IS_ERR(xrcd
)) {
1499 xrcd
->device
= device
;
1501 atomic_set(&xrcd
->usecnt
, 0);
1502 mutex_init(&xrcd
->tgt_qp_mutex
);
1503 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
1508 EXPORT_SYMBOL(ib_alloc_xrcd
);
1510 int ib_dealloc_xrcd(struct ib_xrcd
*xrcd
)
1515 if (atomic_read(&xrcd
->usecnt
))
1518 while (!list_empty(&xrcd
->tgt_qp_list
)) {
1519 qp
= list_entry(xrcd
->tgt_qp_list
.next
, struct ib_qp
, xrcd_list
);
1520 ret
= ib_destroy_qp(qp
);
1525 return xrcd
->device
->dealloc_xrcd(xrcd
);
1527 EXPORT_SYMBOL(ib_dealloc_xrcd
);
1529 struct ib_flow
*ib_create_flow(struct ib_qp
*qp
,
1530 struct ib_flow_attr
*flow_attr
,
1533 struct ib_flow
*flow_id
;
1534 if (!qp
->device
->create_flow
)
1535 return ERR_PTR(-ENOSYS
);
1537 flow_id
= qp
->device
->create_flow(qp
, flow_attr
, domain
);
1538 if (!IS_ERR(flow_id
))
1539 atomic_inc(&qp
->usecnt
);
1542 EXPORT_SYMBOL(ib_create_flow
);
1544 int ib_destroy_flow(struct ib_flow
*flow_id
)
1547 struct ib_qp
*qp
= flow_id
->qp
;
1549 err
= qp
->device
->destroy_flow(flow_id
);
1551 atomic_dec(&qp
->usecnt
);
1554 EXPORT_SYMBOL(ib_destroy_flow
);
1556 int ib_check_mr_status(struct ib_mr
*mr
, u32 check_mask
,
1557 struct ib_mr_status
*mr_status
)
1559 return mr
->device
->check_mr_status
?
1560 mr
->device
->check_mr_status(mr
, check_mask
, mr_status
) : -ENOSYS
;
1562 EXPORT_SYMBOL(ib_check_mr_status
);
1564 int ib_set_vf_link_state(struct ib_device
*device
, int vf
, u8 port
,
1567 if (!device
->set_vf_link_state
)
1570 return device
->set_vf_link_state(device
, vf
, port
, state
);
1572 EXPORT_SYMBOL(ib_set_vf_link_state
);
1574 int ib_get_vf_config(struct ib_device
*device
, int vf
, u8 port
,
1575 struct ifla_vf_info
*info
)
1577 if (!device
->get_vf_config
)
1580 return device
->get_vf_config(device
, vf
, port
, info
);
1582 EXPORT_SYMBOL(ib_get_vf_config
);
1584 int ib_get_vf_stats(struct ib_device
*device
, int vf
, u8 port
,
1585 struct ifla_vf_stats
*stats
)
1587 if (!device
->get_vf_stats
)
1590 return device
->get_vf_stats(device
, vf
, port
, stats
);
1592 EXPORT_SYMBOL(ib_get_vf_stats
);
1594 int ib_set_vf_guid(struct ib_device
*device
, int vf
, u8 port
, u64 guid
,
1597 if (!device
->set_vf_guid
)
1600 return device
->set_vf_guid(device
, vf
, port
, guid
, type
);
1602 EXPORT_SYMBOL(ib_set_vf_guid
);
1605 * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
1606 * and set it the memory region.
1607 * @mr: memory region
1608 * @sg: dma mapped scatterlist
1609 * @sg_nents: number of entries in sg
1610 * @sg_offset: offset in bytes into sg
1611 * @page_size: page vector desired page size
1614 * - The first sg element is allowed to have an offset.
1615 * - Each sg element must be aligned to page_size (or physically
1616 * contiguous to the previous element). In case an sg element has a
1617 * non contiguous offset, the mapping prefix will not include it.
1618 * - The last sg element is allowed to have length less than page_size.
1619 * - If sg_nents total byte length exceeds the mr max_num_sge * page_size
1620 * then only max_num_sg entries will be mapped.
1621 * - If the MR was allocated with type IB_MR_TYPE_SG_GAPS_REG, non of these
1622 * constraints holds and the page_size argument is ignored.
1624 * Returns the number of sg elements that were mapped to the memory region.
1626 * After this completes successfully, the memory region
1627 * is ready for registration.
1629 int ib_map_mr_sg(struct ib_mr
*mr
, struct scatterlist
*sg
, int sg_nents
,
1630 unsigned int sg_offset
, unsigned int page_size
)
1632 if (unlikely(!mr
->device
->map_mr_sg
))
1635 mr
->page_size
= page_size
;
1637 return mr
->device
->map_mr_sg(mr
, sg
, sg_nents
, sg_offset
);
1639 EXPORT_SYMBOL(ib_map_mr_sg
);
1642 * ib_sg_to_pages() - Convert the largest prefix of a sg list
1644 * @mr: memory region
1645 * @sgl: dma mapped scatterlist
1646 * @sg_nents: number of entries in sg
1647 * @sg_offset: offset in bytes into sg
1648 * @set_page: driver page assignment function pointer
1650 * Core service helper for drivers to convert the largest
1651 * prefix of given sg list to a page vector. The sg list
1652 * prefix converted is the prefix that meet the requirements
1655 * Returns the number of sg elements that were assigned to
1658 int ib_sg_to_pages(struct ib_mr
*mr
, struct scatterlist
*sgl
, int sg_nents
,
1659 unsigned int sg_offset
, int (*set_page
)(struct ib_mr
*, u64
))
1661 struct scatterlist
*sg
;
1662 u64 last_end_dma_addr
= 0;
1663 unsigned int last_page_off
= 0;
1664 u64 page_mask
= ~((u64
)mr
->page_size
- 1);
1667 mr
->iova
= sg_dma_address(&sgl
[0]) + sg_offset
;
1670 for_each_sg(sgl
, sg
, sg_nents
, i
) {
1671 u64 dma_addr
= sg_dma_address(sg
) + sg_offset
;
1672 unsigned int dma_len
= sg_dma_len(sg
) - sg_offset
;
1673 u64 end_dma_addr
= dma_addr
+ dma_len
;
1674 u64 page_addr
= dma_addr
& page_mask
;
1677 * For the second and later elements, check whether either the
1678 * end of element i-1 or the start of element i is not aligned
1679 * on a page boundary.
1681 if (i
&& (last_page_off
!= 0 || page_addr
!= dma_addr
)) {
1682 /* Stop mapping if there is a gap. */
1683 if (last_end_dma_addr
!= dma_addr
)
1687 * Coalesce this element with the last. If it is small
1688 * enough just update mr->length. Otherwise start
1689 * mapping from the next page.
1695 ret
= set_page(mr
, page_addr
);
1696 if (unlikely(ret
< 0))
1699 page_addr
+= mr
->page_size
;
1700 } while (page_addr
< end_dma_addr
);
1702 mr
->length
+= dma_len
;
1703 last_end_dma_addr
= end_dma_addr
;
1704 last_page_off
= end_dma_addr
& ~page_mask
;
1711 EXPORT_SYMBOL(ib_sg_to_pages
);
1713 struct ib_drain_cqe
{
1715 struct completion done
;
1718 static void ib_drain_qp_done(struct ib_cq
*cq
, struct ib_wc
*wc
)
1720 struct ib_drain_cqe
*cqe
= container_of(wc
->wr_cqe
, struct ib_drain_cqe
,
1723 complete(&cqe
->done
);
1727 * Post a WR and block until its completion is reaped for the SQ.
1729 static void __ib_drain_sq(struct ib_qp
*qp
)
1731 struct ib_qp_attr attr
= { .qp_state
= IB_QPS_ERR
};
1732 struct ib_drain_cqe sdrain
;
1733 struct ib_send_wr swr
= {}, *bad_swr
;
1736 if (qp
->send_cq
->poll_ctx
== IB_POLL_DIRECT
) {
1737 WARN_ONCE(qp
->send_cq
->poll_ctx
== IB_POLL_DIRECT
,
1738 "IB_POLL_DIRECT poll_ctx not supported for drain\n");
1742 swr
.wr_cqe
= &sdrain
.cqe
;
1743 sdrain
.cqe
.done
= ib_drain_qp_done
;
1744 init_completion(&sdrain
.done
);
1746 ret
= ib_modify_qp(qp
, &attr
, IB_QP_STATE
);
1748 WARN_ONCE(ret
, "failed to drain send queue: %d\n", ret
);
1752 ret
= ib_post_send(qp
, &swr
, &bad_swr
);
1754 WARN_ONCE(ret
, "failed to drain send queue: %d\n", ret
);
1758 wait_for_completion(&sdrain
.done
);
1762 * Post a WR and block until its completion is reaped for the RQ.
1764 static void __ib_drain_rq(struct ib_qp
*qp
)
1766 struct ib_qp_attr attr
= { .qp_state
= IB_QPS_ERR
};
1767 struct ib_drain_cqe rdrain
;
1768 struct ib_recv_wr rwr
= {}, *bad_rwr
;
1771 if (qp
->recv_cq
->poll_ctx
== IB_POLL_DIRECT
) {
1772 WARN_ONCE(qp
->recv_cq
->poll_ctx
== IB_POLL_DIRECT
,
1773 "IB_POLL_DIRECT poll_ctx not supported for drain\n");
1777 rwr
.wr_cqe
= &rdrain
.cqe
;
1778 rdrain
.cqe
.done
= ib_drain_qp_done
;
1779 init_completion(&rdrain
.done
);
1781 ret
= ib_modify_qp(qp
, &attr
, IB_QP_STATE
);
1783 WARN_ONCE(ret
, "failed to drain recv queue: %d\n", ret
);
1787 ret
= ib_post_recv(qp
, &rwr
, &bad_rwr
);
1789 WARN_ONCE(ret
, "failed to drain recv queue: %d\n", ret
);
1793 wait_for_completion(&rdrain
.done
);
1797 * ib_drain_sq() - Block until all SQ CQEs have been consumed by the
1799 * @qp: queue pair to drain
1801 * If the device has a provider-specific drain function, then
1802 * call that. Otherwise call the generic drain function
1807 * ensure there is room in the CQ and SQ for the drain work request and
1810 * allocate the CQ using ib_alloc_cq() and the CQ poll context cannot be
1813 * ensure that there are no other contexts that are posting WRs concurrently.
1814 * Otherwise the drain is not guaranteed.
1816 void ib_drain_sq(struct ib_qp
*qp
)
1818 if (qp
->device
->drain_sq
)
1819 qp
->device
->drain_sq(qp
);
1823 EXPORT_SYMBOL(ib_drain_sq
);
1826 * ib_drain_rq() - Block until all RQ CQEs have been consumed by the
1828 * @qp: queue pair to drain
1830 * If the device has a provider-specific drain function, then
1831 * call that. Otherwise call the generic drain function
1836 * ensure there is room in the CQ and RQ for the drain work request and
1839 * allocate the CQ using ib_alloc_cq() and the CQ poll context cannot be
1842 * ensure that there are no other contexts that are posting WRs concurrently.
1843 * Otherwise the drain is not guaranteed.
1845 void ib_drain_rq(struct ib_qp
*qp
)
1847 if (qp
->device
->drain_rq
)
1848 qp
->device
->drain_rq(qp
);
1852 EXPORT_SYMBOL(ib_drain_rq
);
1855 * ib_drain_qp() - Block until all CQEs have been consumed by the
1856 * application on both the RQ and SQ.
1857 * @qp: queue pair to drain
1861 * ensure there is room in the CQ(s), SQ, and RQ for drain work requests
1864 * allocate the CQs using ib_alloc_cq() and the CQ poll context cannot be
1867 * ensure that there are no other contexts that are posting WRs concurrently.
1868 * Otherwise the drain is not guaranteed.
1870 void ib_drain_qp(struct ib_qp
*qp
)
1876 EXPORT_SYMBOL(ib_drain_qp
);