2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/file.h>
38 #include <linux/slab.h>
40 #include <asm/uaccess.h>
44 static struct lock_class_key pd_lock_key
;
45 static struct lock_class_key mr_lock_key
;
46 static struct lock_class_key cq_lock_key
;
47 static struct lock_class_key qp_lock_key
;
48 static struct lock_class_key ah_lock_key
;
49 static struct lock_class_key srq_lock_key
;
51 #define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \
53 (udata)->inbuf = (void __user *) (ibuf); \
54 (udata)->outbuf = (void __user *) (obuf); \
55 (udata)->inlen = (ilen); \
56 (udata)->outlen = (olen); \
60 * The ib_uobject locking scheme is as follows:
62 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it
63 * needs to be held during all idr operations. When an object is
64 * looked up, a reference must be taken on the object's kref before
67 * - Each object also has an rwsem. This rwsem must be held for
68 * reading while an operation that uses the object is performed.
69 * For example, while registering an MR, the associated PD's
70 * uobject.mutex must be held for reading. The rwsem must be held
71 * for writing while initializing or destroying an object.
73 * - In addition, each object has a "live" flag. If this flag is not
74 * set, then lookups of the object will fail even if it is found in
75 * the idr. This handles a reader that blocks and does not acquire
76 * the rwsem until after the object is destroyed. The destroy
77 * operation will set the live flag to 0 and then drop the rwsem;
78 * this will allow the reader to acquire the rwsem, see that the
79 * live flag is 0, and then drop the rwsem and its reference to
80 * object. The underlying storage will not be freed until the last
81 * reference to the object is dropped.
84 static void init_uobj(struct ib_uobject
*uobj
, u64 user_handle
,
85 struct ib_ucontext
*context
, struct lock_class_key
*key
)
87 uobj
->user_handle
= user_handle
;
88 uobj
->context
= context
;
89 kref_init(&uobj
->ref
);
90 init_rwsem(&uobj
->mutex
);
91 lockdep_set_class(&uobj
->mutex
, key
);
95 static void release_uobj(struct kref
*kref
)
97 kfree(container_of(kref
, struct ib_uobject
, ref
));
100 static void put_uobj(struct ib_uobject
*uobj
)
102 kref_put(&uobj
->ref
, release_uobj
);
105 static void put_uobj_read(struct ib_uobject
*uobj
)
107 up_read(&uobj
->mutex
);
111 static void put_uobj_write(struct ib_uobject
*uobj
)
113 up_write(&uobj
->mutex
);
117 static int idr_add_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
122 if (!idr_pre_get(idr
, GFP_KERNEL
))
125 spin_lock(&ib_uverbs_idr_lock
);
126 ret
= idr_get_new(idr
, uobj
, &uobj
->id
);
127 spin_unlock(&ib_uverbs_idr_lock
);
135 void idr_remove_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
137 spin_lock(&ib_uverbs_idr_lock
);
138 idr_remove(idr
, uobj
->id
);
139 spin_unlock(&ib_uverbs_idr_lock
);
142 static struct ib_uobject
*__idr_get_uobj(struct idr
*idr
, int id
,
143 struct ib_ucontext
*context
)
145 struct ib_uobject
*uobj
;
147 spin_lock(&ib_uverbs_idr_lock
);
148 uobj
= idr_find(idr
, id
);
150 if (uobj
->context
== context
)
151 kref_get(&uobj
->ref
);
155 spin_unlock(&ib_uverbs_idr_lock
);
160 static struct ib_uobject
*idr_read_uobj(struct idr
*idr
, int id
,
161 struct ib_ucontext
*context
, int nested
)
163 struct ib_uobject
*uobj
;
165 uobj
= __idr_get_uobj(idr
, id
, context
);
170 down_read_nested(&uobj
->mutex
, SINGLE_DEPTH_NESTING
);
172 down_read(&uobj
->mutex
);
181 static struct ib_uobject
*idr_write_uobj(struct idr
*idr
, int id
,
182 struct ib_ucontext
*context
)
184 struct ib_uobject
*uobj
;
186 uobj
= __idr_get_uobj(idr
, id
, context
);
190 down_write(&uobj
->mutex
);
192 put_uobj_write(uobj
);
199 static void *idr_read_obj(struct idr
*idr
, int id
, struct ib_ucontext
*context
,
202 struct ib_uobject
*uobj
;
204 uobj
= idr_read_uobj(idr
, id
, context
, nested
);
205 return uobj
? uobj
->object
: NULL
;
208 static struct ib_pd
*idr_read_pd(int pd_handle
, struct ib_ucontext
*context
)
210 return idr_read_obj(&ib_uverbs_pd_idr
, pd_handle
, context
, 0);
213 static void put_pd_read(struct ib_pd
*pd
)
215 put_uobj_read(pd
->uobject
);
218 static struct ib_cq
*idr_read_cq(int cq_handle
, struct ib_ucontext
*context
, int nested
)
220 return idr_read_obj(&ib_uverbs_cq_idr
, cq_handle
, context
, nested
);
223 static void put_cq_read(struct ib_cq
*cq
)
225 put_uobj_read(cq
->uobject
);
228 static struct ib_ah
*idr_read_ah(int ah_handle
, struct ib_ucontext
*context
)
230 return idr_read_obj(&ib_uverbs_ah_idr
, ah_handle
, context
, 0);
233 static void put_ah_read(struct ib_ah
*ah
)
235 put_uobj_read(ah
->uobject
);
238 static struct ib_qp
*idr_read_qp(int qp_handle
, struct ib_ucontext
*context
)
240 return idr_read_obj(&ib_uverbs_qp_idr
, qp_handle
, context
, 0);
243 static void put_qp_read(struct ib_qp
*qp
)
245 put_uobj_read(qp
->uobject
);
248 static struct ib_srq
*idr_read_srq(int srq_handle
, struct ib_ucontext
*context
)
250 return idr_read_obj(&ib_uverbs_srq_idr
, srq_handle
, context
, 0);
253 static void put_srq_read(struct ib_srq
*srq
)
255 put_uobj_read(srq
->uobject
);
258 ssize_t
ib_uverbs_get_context(struct ib_uverbs_file
*file
,
259 const char __user
*buf
,
260 int in_len
, int out_len
)
262 struct ib_uverbs_get_context cmd
;
263 struct ib_uverbs_get_context_resp resp
;
264 struct ib_udata udata
;
265 struct ib_device
*ibdev
= file
->device
->ib_dev
;
266 struct ib_ucontext
*ucontext
;
270 if (out_len
< sizeof resp
)
273 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
276 mutex_lock(&file
->mutex
);
278 if (file
->ucontext
) {
283 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
284 (unsigned long) cmd
.response
+ sizeof resp
,
285 in_len
- sizeof cmd
, out_len
- sizeof resp
);
287 ucontext
= ibdev
->alloc_ucontext(ibdev
, &udata
);
288 if (IS_ERR(ucontext
)) {
289 ret
= PTR_ERR(ucontext
);
293 ucontext
->device
= ibdev
;
294 INIT_LIST_HEAD(&ucontext
->pd_list
);
295 INIT_LIST_HEAD(&ucontext
->mr_list
);
296 INIT_LIST_HEAD(&ucontext
->mw_list
);
297 INIT_LIST_HEAD(&ucontext
->cq_list
);
298 INIT_LIST_HEAD(&ucontext
->qp_list
);
299 INIT_LIST_HEAD(&ucontext
->srq_list
);
300 INIT_LIST_HEAD(&ucontext
->ah_list
);
301 ucontext
->closing
= 0;
303 resp
.num_comp_vectors
= file
->device
->num_comp_vectors
;
305 ret
= get_unused_fd();
310 filp
= ib_uverbs_alloc_event_file(file
, 1);
316 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
317 &resp
, sizeof resp
)) {
322 file
->async_file
= filp
->private_data
;
324 INIT_IB_EVENT_HANDLER(&file
->event_handler
, file
->device
->ib_dev
,
325 ib_uverbs_event_handler
);
326 ret
= ib_register_event_handler(&file
->event_handler
);
330 kref_get(&file
->async_file
->ref
);
331 kref_get(&file
->ref
);
332 file
->ucontext
= ucontext
;
334 fd_install(resp
.async_fd
, filp
);
336 mutex_unlock(&file
->mutex
);
344 put_unused_fd(resp
.async_fd
);
347 ibdev
->dealloc_ucontext(ucontext
);
350 mutex_unlock(&file
->mutex
);
354 ssize_t
ib_uverbs_query_device(struct ib_uverbs_file
*file
,
355 const char __user
*buf
,
356 int in_len
, int out_len
)
358 struct ib_uverbs_query_device cmd
;
359 struct ib_uverbs_query_device_resp resp
;
360 struct ib_device_attr attr
;
363 if (out_len
< sizeof resp
)
366 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
369 ret
= ib_query_device(file
->device
->ib_dev
, &attr
);
373 memset(&resp
, 0, sizeof resp
);
375 resp
.fw_ver
= attr
.fw_ver
;
376 resp
.node_guid
= file
->device
->ib_dev
->node_guid
;
377 resp
.sys_image_guid
= attr
.sys_image_guid
;
378 resp
.max_mr_size
= attr
.max_mr_size
;
379 resp
.page_size_cap
= attr
.page_size_cap
;
380 resp
.vendor_id
= attr
.vendor_id
;
381 resp
.vendor_part_id
= attr
.vendor_part_id
;
382 resp
.hw_ver
= attr
.hw_ver
;
383 resp
.max_qp
= attr
.max_qp
;
384 resp
.max_qp_wr
= attr
.max_qp_wr
;
385 resp
.device_cap_flags
= attr
.device_cap_flags
;
386 resp
.max_sge
= attr
.max_sge
;
387 resp
.max_sge_rd
= attr
.max_sge_rd
;
388 resp
.max_cq
= attr
.max_cq
;
389 resp
.max_cqe
= attr
.max_cqe
;
390 resp
.max_mr
= attr
.max_mr
;
391 resp
.max_pd
= attr
.max_pd
;
392 resp
.max_qp_rd_atom
= attr
.max_qp_rd_atom
;
393 resp
.max_ee_rd_atom
= attr
.max_ee_rd_atom
;
394 resp
.max_res_rd_atom
= attr
.max_res_rd_atom
;
395 resp
.max_qp_init_rd_atom
= attr
.max_qp_init_rd_atom
;
396 resp
.max_ee_init_rd_atom
= attr
.max_ee_init_rd_atom
;
397 resp
.atomic_cap
= attr
.atomic_cap
;
398 resp
.max_ee
= attr
.max_ee
;
399 resp
.max_rdd
= attr
.max_rdd
;
400 resp
.max_mw
= attr
.max_mw
;
401 resp
.max_raw_ipv6_qp
= attr
.max_raw_ipv6_qp
;
402 resp
.max_raw_ethy_qp
= attr
.max_raw_ethy_qp
;
403 resp
.max_mcast_grp
= attr
.max_mcast_grp
;
404 resp
.max_mcast_qp_attach
= attr
.max_mcast_qp_attach
;
405 resp
.max_total_mcast_qp_attach
= attr
.max_total_mcast_qp_attach
;
406 resp
.max_ah
= attr
.max_ah
;
407 resp
.max_fmr
= attr
.max_fmr
;
408 resp
.max_map_per_fmr
= attr
.max_map_per_fmr
;
409 resp
.max_srq
= attr
.max_srq
;
410 resp
.max_srq_wr
= attr
.max_srq_wr
;
411 resp
.max_srq_sge
= attr
.max_srq_sge
;
412 resp
.max_pkeys
= attr
.max_pkeys
;
413 resp
.local_ca_ack_delay
= attr
.local_ca_ack_delay
;
414 resp
.phys_port_cnt
= file
->device
->ib_dev
->phys_port_cnt
;
416 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
423 ssize_t
ib_uverbs_query_port(struct ib_uverbs_file
*file
,
424 const char __user
*buf
,
425 int in_len
, int out_len
)
427 struct ib_uverbs_query_port cmd
;
428 struct ib_uverbs_query_port_resp resp
;
429 struct ib_port_attr attr
;
432 if (out_len
< sizeof resp
)
435 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
438 ret
= ib_query_port(file
->device
->ib_dev
, cmd
.port_num
, &attr
);
442 memset(&resp
, 0, sizeof resp
);
444 resp
.state
= attr
.state
;
445 resp
.max_mtu
= attr
.max_mtu
;
446 resp
.active_mtu
= attr
.active_mtu
;
447 resp
.gid_tbl_len
= attr
.gid_tbl_len
;
448 resp
.port_cap_flags
= attr
.port_cap_flags
;
449 resp
.max_msg_sz
= attr
.max_msg_sz
;
450 resp
.bad_pkey_cntr
= attr
.bad_pkey_cntr
;
451 resp
.qkey_viol_cntr
= attr
.qkey_viol_cntr
;
452 resp
.pkey_tbl_len
= attr
.pkey_tbl_len
;
454 resp
.sm_lid
= attr
.sm_lid
;
456 resp
.max_vl_num
= attr
.max_vl_num
;
457 resp
.sm_sl
= attr
.sm_sl
;
458 resp
.subnet_timeout
= attr
.subnet_timeout
;
459 resp
.init_type_reply
= attr
.init_type_reply
;
460 resp
.active_width
= attr
.active_width
;
461 resp
.active_speed
= attr
.active_speed
;
462 resp
.phys_state
= attr
.phys_state
;
464 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
471 ssize_t
ib_uverbs_alloc_pd(struct ib_uverbs_file
*file
,
472 const char __user
*buf
,
473 int in_len
, int out_len
)
475 struct ib_uverbs_alloc_pd cmd
;
476 struct ib_uverbs_alloc_pd_resp resp
;
477 struct ib_udata udata
;
478 struct ib_uobject
*uobj
;
482 if (out_len
< sizeof resp
)
485 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
488 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
489 (unsigned long) cmd
.response
+ sizeof resp
,
490 in_len
- sizeof cmd
, out_len
- sizeof resp
);
492 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
496 init_uobj(uobj
, 0, file
->ucontext
, &pd_lock_key
);
497 down_write(&uobj
->mutex
);
499 pd
= file
->device
->ib_dev
->alloc_pd(file
->device
->ib_dev
,
500 file
->ucontext
, &udata
);
506 pd
->device
= file
->device
->ib_dev
;
508 atomic_set(&pd
->usecnt
, 0);
511 ret
= idr_add_uobj(&ib_uverbs_pd_idr
, uobj
);
515 memset(&resp
, 0, sizeof resp
);
516 resp
.pd_handle
= uobj
->id
;
518 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
519 &resp
, sizeof resp
)) {
524 mutex_lock(&file
->mutex
);
525 list_add_tail(&uobj
->list
, &file
->ucontext
->pd_list
);
526 mutex_unlock(&file
->mutex
);
530 up_write(&uobj
->mutex
);
535 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
541 put_uobj_write(uobj
);
545 ssize_t
ib_uverbs_dealloc_pd(struct ib_uverbs_file
*file
,
546 const char __user
*buf
,
547 int in_len
, int out_len
)
549 struct ib_uverbs_dealloc_pd cmd
;
550 struct ib_uobject
*uobj
;
553 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
556 uobj
= idr_write_uobj(&ib_uverbs_pd_idr
, cmd
.pd_handle
, file
->ucontext
);
560 ret
= ib_dealloc_pd(uobj
->object
);
564 put_uobj_write(uobj
);
569 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
571 mutex_lock(&file
->mutex
);
572 list_del(&uobj
->list
);
573 mutex_unlock(&file
->mutex
);
580 ssize_t
ib_uverbs_reg_mr(struct ib_uverbs_file
*file
,
581 const char __user
*buf
, int in_len
,
584 struct ib_uverbs_reg_mr cmd
;
585 struct ib_uverbs_reg_mr_resp resp
;
586 struct ib_udata udata
;
587 struct ib_uobject
*uobj
;
592 if (out_len
< sizeof resp
)
595 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
598 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
599 (unsigned long) cmd
.response
+ sizeof resp
,
600 in_len
- sizeof cmd
, out_len
- sizeof resp
);
602 if ((cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
))
606 * Local write permission is required if remote write or
607 * remote atomic permission is also requested.
609 if (cmd
.access_flags
& (IB_ACCESS_REMOTE_ATOMIC
| IB_ACCESS_REMOTE_WRITE
) &&
610 !(cmd
.access_flags
& IB_ACCESS_LOCAL_WRITE
))
613 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
617 init_uobj(uobj
, 0, file
->ucontext
, &mr_lock_key
);
618 down_write(&uobj
->mutex
);
620 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
626 mr
= pd
->device
->reg_user_mr(pd
, cmd
.start
, cmd
.length
, cmd
.hca_va
,
627 cmd
.access_flags
, &udata
);
633 mr
->device
= pd
->device
;
636 atomic_inc(&pd
->usecnt
);
637 atomic_set(&mr
->usecnt
, 0);
640 ret
= idr_add_uobj(&ib_uverbs_mr_idr
, uobj
);
644 memset(&resp
, 0, sizeof resp
);
645 resp
.lkey
= mr
->lkey
;
646 resp
.rkey
= mr
->rkey
;
647 resp
.mr_handle
= uobj
->id
;
649 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
650 &resp
, sizeof resp
)) {
657 mutex_lock(&file
->mutex
);
658 list_add_tail(&uobj
->list
, &file
->ucontext
->mr_list
);
659 mutex_unlock(&file
->mutex
);
663 up_write(&uobj
->mutex
);
668 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
677 put_uobj_write(uobj
);
681 ssize_t
ib_uverbs_dereg_mr(struct ib_uverbs_file
*file
,
682 const char __user
*buf
, int in_len
,
685 struct ib_uverbs_dereg_mr cmd
;
687 struct ib_uobject
*uobj
;
690 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
693 uobj
= idr_write_uobj(&ib_uverbs_mr_idr
, cmd
.mr_handle
, file
->ucontext
);
699 ret
= ib_dereg_mr(mr
);
703 put_uobj_write(uobj
);
708 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
710 mutex_lock(&file
->mutex
);
711 list_del(&uobj
->list
);
712 mutex_unlock(&file
->mutex
);
719 ssize_t
ib_uverbs_create_comp_channel(struct ib_uverbs_file
*file
,
720 const char __user
*buf
, int in_len
,
723 struct ib_uverbs_create_comp_channel cmd
;
724 struct ib_uverbs_create_comp_channel_resp resp
;
728 if (out_len
< sizeof resp
)
731 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
734 ret
= get_unused_fd();
739 filp
= ib_uverbs_alloc_event_file(file
, 0);
741 put_unused_fd(resp
.fd
);
742 return PTR_ERR(filp
);
745 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
746 &resp
, sizeof resp
)) {
747 put_unused_fd(resp
.fd
);
752 fd_install(resp
.fd
, filp
);
756 ssize_t
ib_uverbs_create_cq(struct ib_uverbs_file
*file
,
757 const char __user
*buf
, int in_len
,
760 struct ib_uverbs_create_cq cmd
;
761 struct ib_uverbs_create_cq_resp resp
;
762 struct ib_udata udata
;
763 struct ib_ucq_object
*obj
;
764 struct ib_uverbs_event_file
*ev_file
= NULL
;
768 if (out_len
< sizeof resp
)
771 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
774 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
775 (unsigned long) cmd
.response
+ sizeof resp
,
776 in_len
- sizeof cmd
, out_len
- sizeof resp
);
778 if (cmd
.comp_vector
>= file
->device
->num_comp_vectors
)
781 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
785 init_uobj(&obj
->uobject
, cmd
.user_handle
, file
->ucontext
, &cq_lock_key
);
786 down_write(&obj
->uobject
.mutex
);
788 if (cmd
.comp_channel
>= 0) {
789 ev_file
= ib_uverbs_lookup_comp_file(cmd
.comp_channel
);
796 obj
->uverbs_file
= file
;
797 obj
->comp_events_reported
= 0;
798 obj
->async_events_reported
= 0;
799 INIT_LIST_HEAD(&obj
->comp_list
);
800 INIT_LIST_HEAD(&obj
->async_list
);
802 cq
= file
->device
->ib_dev
->create_cq(file
->device
->ib_dev
, cmd
.cqe
,
804 file
->ucontext
, &udata
);
810 cq
->device
= file
->device
->ib_dev
;
811 cq
->uobject
= &obj
->uobject
;
812 cq
->comp_handler
= ib_uverbs_comp_handler
;
813 cq
->event_handler
= ib_uverbs_cq_event_handler
;
814 cq
->cq_context
= ev_file
;
815 atomic_set(&cq
->usecnt
, 0);
817 obj
->uobject
.object
= cq
;
818 ret
= idr_add_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
822 memset(&resp
, 0, sizeof resp
);
823 resp
.cq_handle
= obj
->uobject
.id
;
826 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
827 &resp
, sizeof resp
)) {
832 mutex_lock(&file
->mutex
);
833 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->cq_list
);
834 mutex_unlock(&file
->mutex
);
836 obj
->uobject
.live
= 1;
838 up_write(&obj
->uobject
.mutex
);
843 idr_remove_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
850 ib_uverbs_release_ucq(file
, ev_file
, obj
);
853 put_uobj_write(&obj
->uobject
);
857 ssize_t
ib_uverbs_resize_cq(struct ib_uverbs_file
*file
,
858 const char __user
*buf
, int in_len
,
861 struct ib_uverbs_resize_cq cmd
;
862 struct ib_uverbs_resize_cq_resp resp
;
863 struct ib_udata udata
;
867 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
870 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
871 (unsigned long) cmd
.response
+ sizeof resp
,
872 in_len
- sizeof cmd
, out_len
- sizeof resp
);
874 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
878 ret
= cq
->device
->resize_cq(cq
, cmd
.cqe
, &udata
);
884 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
885 &resp
, sizeof resp
.cqe
))
891 return ret
? ret
: in_len
;
894 static int copy_wc_to_user(void __user
*dest
, struct ib_wc
*wc
)
896 struct ib_uverbs_wc tmp
;
898 tmp
.wr_id
= wc
->wr_id
;
899 tmp
.status
= wc
->status
;
900 tmp
.opcode
= wc
->opcode
;
901 tmp
.vendor_err
= wc
->vendor_err
;
902 tmp
.byte_len
= wc
->byte_len
;
903 tmp
.ex
.imm_data
= (__u32 __force
) wc
->ex
.imm_data
;
904 tmp
.qp_num
= wc
->qp
->qp_num
;
905 tmp
.src_qp
= wc
->src_qp
;
906 tmp
.wc_flags
= wc
->wc_flags
;
907 tmp
.pkey_index
= wc
->pkey_index
;
910 tmp
.dlid_path_bits
= wc
->dlid_path_bits
;
911 tmp
.port_num
= wc
->port_num
;
914 if (copy_to_user(dest
, &tmp
, sizeof tmp
))
920 ssize_t
ib_uverbs_poll_cq(struct ib_uverbs_file
*file
,
921 const char __user
*buf
, int in_len
,
924 struct ib_uverbs_poll_cq cmd
;
925 struct ib_uverbs_poll_cq_resp resp
;
926 u8 __user
*header_ptr
;
932 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
935 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
939 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
940 header_ptr
= (void __user
*)(unsigned long) cmd
.response
;
941 data_ptr
= header_ptr
+ sizeof resp
;
943 memset(&resp
, 0, sizeof resp
);
944 while (resp
.count
< cmd
.ne
) {
945 ret
= ib_poll_cq(cq
, 1, &wc
);
951 ret
= copy_wc_to_user(data_ptr
, &wc
);
955 data_ptr
+= sizeof(struct ib_uverbs_wc
);
959 if (copy_to_user(header_ptr
, &resp
, sizeof resp
)) {
971 ssize_t
ib_uverbs_req_notify_cq(struct ib_uverbs_file
*file
,
972 const char __user
*buf
, int in_len
,
975 struct ib_uverbs_req_notify_cq cmd
;
978 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
981 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
985 ib_req_notify_cq(cq
, cmd
.solicited_only
?
986 IB_CQ_SOLICITED
: IB_CQ_NEXT_COMP
);
993 ssize_t
ib_uverbs_destroy_cq(struct ib_uverbs_file
*file
,
994 const char __user
*buf
, int in_len
,
997 struct ib_uverbs_destroy_cq cmd
;
998 struct ib_uverbs_destroy_cq_resp resp
;
999 struct ib_uobject
*uobj
;
1001 struct ib_ucq_object
*obj
;
1002 struct ib_uverbs_event_file
*ev_file
;
1005 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1008 uobj
= idr_write_uobj(&ib_uverbs_cq_idr
, cmd
.cq_handle
, file
->ucontext
);
1012 ev_file
= cq
->cq_context
;
1013 obj
= container_of(cq
->uobject
, struct ib_ucq_object
, uobject
);
1015 ret
= ib_destroy_cq(cq
);
1019 put_uobj_write(uobj
);
1024 idr_remove_uobj(&ib_uverbs_cq_idr
, uobj
);
1026 mutex_lock(&file
->mutex
);
1027 list_del(&uobj
->list
);
1028 mutex_unlock(&file
->mutex
);
1030 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1032 memset(&resp
, 0, sizeof resp
);
1033 resp
.comp_events_reported
= obj
->comp_events_reported
;
1034 resp
.async_events_reported
= obj
->async_events_reported
;
1038 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1039 &resp
, sizeof resp
))
1045 ssize_t
ib_uverbs_create_qp(struct ib_uverbs_file
*file
,
1046 const char __user
*buf
, int in_len
,
1049 struct ib_uverbs_create_qp cmd
;
1050 struct ib_uverbs_create_qp_resp resp
;
1051 struct ib_udata udata
;
1052 struct ib_uqp_object
*obj
;
1054 struct ib_cq
*scq
, *rcq
;
1057 struct ib_qp_init_attr attr
;
1060 if (out_len
< sizeof resp
)
1063 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1066 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1067 (unsigned long) cmd
.response
+ sizeof resp
,
1068 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1070 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
1074 init_uobj(&obj
->uevent
.uobject
, cmd
.user_handle
, file
->ucontext
, &qp_lock_key
);
1075 down_write(&obj
->uevent
.uobject
.mutex
);
1077 srq
= cmd
.is_srq
? idr_read_srq(cmd
.srq_handle
, file
->ucontext
) : NULL
;
1078 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1079 scq
= idr_read_cq(cmd
.send_cq_handle
, file
->ucontext
, 0);
1080 rcq
= cmd
.recv_cq_handle
== cmd
.send_cq_handle
?
1081 scq
: idr_read_cq(cmd
.recv_cq_handle
, file
->ucontext
, 1);
1083 if (!pd
|| !scq
|| !rcq
|| (cmd
.is_srq
&& !srq
)) {
1088 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1089 attr
.qp_context
= file
;
1093 attr
.sq_sig_type
= cmd
.sq_sig_all
? IB_SIGNAL_ALL_WR
: IB_SIGNAL_REQ_WR
;
1094 attr
.qp_type
= cmd
.qp_type
;
1095 attr
.create_flags
= 0;
1097 attr
.cap
.max_send_wr
= cmd
.max_send_wr
;
1098 attr
.cap
.max_recv_wr
= cmd
.max_recv_wr
;
1099 attr
.cap
.max_send_sge
= cmd
.max_send_sge
;
1100 attr
.cap
.max_recv_sge
= cmd
.max_recv_sge
;
1101 attr
.cap
.max_inline_data
= cmd
.max_inline_data
;
1103 obj
->uevent
.events_reported
= 0;
1104 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1105 INIT_LIST_HEAD(&obj
->mcast_list
);
1107 qp
= pd
->device
->create_qp(pd
, &attr
, &udata
);
1113 qp
->device
= pd
->device
;
1115 qp
->send_cq
= attr
.send_cq
;
1116 qp
->recv_cq
= attr
.recv_cq
;
1118 qp
->uobject
= &obj
->uevent
.uobject
;
1119 qp
->event_handler
= attr
.event_handler
;
1120 qp
->qp_context
= attr
.qp_context
;
1121 qp
->qp_type
= attr
.qp_type
;
1122 atomic_inc(&pd
->usecnt
);
1123 atomic_inc(&attr
.send_cq
->usecnt
);
1124 atomic_inc(&attr
.recv_cq
->usecnt
);
1126 atomic_inc(&attr
.srq
->usecnt
);
1128 obj
->uevent
.uobject
.object
= qp
;
1129 ret
= idr_add_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1133 memset(&resp
, 0, sizeof resp
);
1134 resp
.qpn
= qp
->qp_num
;
1135 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1136 resp
.max_recv_sge
= attr
.cap
.max_recv_sge
;
1137 resp
.max_send_sge
= attr
.cap
.max_send_sge
;
1138 resp
.max_recv_wr
= attr
.cap
.max_recv_wr
;
1139 resp
.max_send_wr
= attr
.cap
.max_send_wr
;
1140 resp
.max_inline_data
= attr
.cap
.max_inline_data
;
1142 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1143 &resp
, sizeof resp
)) {
1155 mutex_lock(&file
->mutex
);
1156 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->qp_list
);
1157 mutex_unlock(&file
->mutex
);
1159 obj
->uevent
.uobject
.live
= 1;
1161 up_write(&obj
->uevent
.uobject
.mutex
);
1166 idr_remove_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1176 if (rcq
&& rcq
!= scq
)
1181 put_uobj_write(&obj
->uevent
.uobject
);
1185 ssize_t
ib_uverbs_query_qp(struct ib_uverbs_file
*file
,
1186 const char __user
*buf
, int in_len
,
1189 struct ib_uverbs_query_qp cmd
;
1190 struct ib_uverbs_query_qp_resp resp
;
1192 struct ib_qp_attr
*attr
;
1193 struct ib_qp_init_attr
*init_attr
;
1196 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1199 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1200 init_attr
= kmalloc(sizeof *init_attr
, GFP_KERNEL
);
1201 if (!attr
|| !init_attr
) {
1206 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1212 ret
= ib_query_qp(qp
, attr
, cmd
.attr_mask
, init_attr
);
1219 memset(&resp
, 0, sizeof resp
);
1221 resp
.qp_state
= attr
->qp_state
;
1222 resp
.cur_qp_state
= attr
->cur_qp_state
;
1223 resp
.path_mtu
= attr
->path_mtu
;
1224 resp
.path_mig_state
= attr
->path_mig_state
;
1225 resp
.qkey
= attr
->qkey
;
1226 resp
.rq_psn
= attr
->rq_psn
;
1227 resp
.sq_psn
= attr
->sq_psn
;
1228 resp
.dest_qp_num
= attr
->dest_qp_num
;
1229 resp
.qp_access_flags
= attr
->qp_access_flags
;
1230 resp
.pkey_index
= attr
->pkey_index
;
1231 resp
.alt_pkey_index
= attr
->alt_pkey_index
;
1232 resp
.sq_draining
= attr
->sq_draining
;
1233 resp
.max_rd_atomic
= attr
->max_rd_atomic
;
1234 resp
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
1235 resp
.min_rnr_timer
= attr
->min_rnr_timer
;
1236 resp
.port_num
= attr
->port_num
;
1237 resp
.timeout
= attr
->timeout
;
1238 resp
.retry_cnt
= attr
->retry_cnt
;
1239 resp
.rnr_retry
= attr
->rnr_retry
;
1240 resp
.alt_port_num
= attr
->alt_port_num
;
1241 resp
.alt_timeout
= attr
->alt_timeout
;
1243 memcpy(resp
.dest
.dgid
, attr
->ah_attr
.grh
.dgid
.raw
, 16);
1244 resp
.dest
.flow_label
= attr
->ah_attr
.grh
.flow_label
;
1245 resp
.dest
.sgid_index
= attr
->ah_attr
.grh
.sgid_index
;
1246 resp
.dest
.hop_limit
= attr
->ah_attr
.grh
.hop_limit
;
1247 resp
.dest
.traffic_class
= attr
->ah_attr
.grh
.traffic_class
;
1248 resp
.dest
.dlid
= attr
->ah_attr
.dlid
;
1249 resp
.dest
.sl
= attr
->ah_attr
.sl
;
1250 resp
.dest
.src_path_bits
= attr
->ah_attr
.src_path_bits
;
1251 resp
.dest
.static_rate
= attr
->ah_attr
.static_rate
;
1252 resp
.dest
.is_global
= !!(attr
->ah_attr
.ah_flags
& IB_AH_GRH
);
1253 resp
.dest
.port_num
= attr
->ah_attr
.port_num
;
1255 memcpy(resp
.alt_dest
.dgid
, attr
->alt_ah_attr
.grh
.dgid
.raw
, 16);
1256 resp
.alt_dest
.flow_label
= attr
->alt_ah_attr
.grh
.flow_label
;
1257 resp
.alt_dest
.sgid_index
= attr
->alt_ah_attr
.grh
.sgid_index
;
1258 resp
.alt_dest
.hop_limit
= attr
->alt_ah_attr
.grh
.hop_limit
;
1259 resp
.alt_dest
.traffic_class
= attr
->alt_ah_attr
.grh
.traffic_class
;
1260 resp
.alt_dest
.dlid
= attr
->alt_ah_attr
.dlid
;
1261 resp
.alt_dest
.sl
= attr
->alt_ah_attr
.sl
;
1262 resp
.alt_dest
.src_path_bits
= attr
->alt_ah_attr
.src_path_bits
;
1263 resp
.alt_dest
.static_rate
= attr
->alt_ah_attr
.static_rate
;
1264 resp
.alt_dest
.is_global
= !!(attr
->alt_ah_attr
.ah_flags
& IB_AH_GRH
);
1265 resp
.alt_dest
.port_num
= attr
->alt_ah_attr
.port_num
;
1267 resp
.max_send_wr
= init_attr
->cap
.max_send_wr
;
1268 resp
.max_recv_wr
= init_attr
->cap
.max_recv_wr
;
1269 resp
.max_send_sge
= init_attr
->cap
.max_send_sge
;
1270 resp
.max_recv_sge
= init_attr
->cap
.max_recv_sge
;
1271 resp
.max_inline_data
= init_attr
->cap
.max_inline_data
;
1272 resp
.sq_sig_all
= init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
1274 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1275 &resp
, sizeof resp
))
1282 return ret
? ret
: in_len
;
1285 ssize_t
ib_uverbs_modify_qp(struct ib_uverbs_file
*file
,
1286 const char __user
*buf
, int in_len
,
1289 struct ib_uverbs_modify_qp cmd
;
1290 struct ib_udata udata
;
1292 struct ib_qp_attr
*attr
;
1295 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1298 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
1301 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1305 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1311 attr
->qp_state
= cmd
.qp_state
;
1312 attr
->cur_qp_state
= cmd
.cur_qp_state
;
1313 attr
->path_mtu
= cmd
.path_mtu
;
1314 attr
->path_mig_state
= cmd
.path_mig_state
;
1315 attr
->qkey
= cmd
.qkey
;
1316 attr
->rq_psn
= cmd
.rq_psn
;
1317 attr
->sq_psn
= cmd
.sq_psn
;
1318 attr
->dest_qp_num
= cmd
.dest_qp_num
;
1319 attr
->qp_access_flags
= cmd
.qp_access_flags
;
1320 attr
->pkey_index
= cmd
.pkey_index
;
1321 attr
->alt_pkey_index
= cmd
.alt_pkey_index
;
1322 attr
->en_sqd_async_notify
= cmd
.en_sqd_async_notify
;
1323 attr
->max_rd_atomic
= cmd
.max_rd_atomic
;
1324 attr
->max_dest_rd_atomic
= cmd
.max_dest_rd_atomic
;
1325 attr
->min_rnr_timer
= cmd
.min_rnr_timer
;
1326 attr
->port_num
= cmd
.port_num
;
1327 attr
->timeout
= cmd
.timeout
;
1328 attr
->retry_cnt
= cmd
.retry_cnt
;
1329 attr
->rnr_retry
= cmd
.rnr_retry
;
1330 attr
->alt_port_num
= cmd
.alt_port_num
;
1331 attr
->alt_timeout
= cmd
.alt_timeout
;
1333 memcpy(attr
->ah_attr
.grh
.dgid
.raw
, cmd
.dest
.dgid
, 16);
1334 attr
->ah_attr
.grh
.flow_label
= cmd
.dest
.flow_label
;
1335 attr
->ah_attr
.grh
.sgid_index
= cmd
.dest
.sgid_index
;
1336 attr
->ah_attr
.grh
.hop_limit
= cmd
.dest
.hop_limit
;
1337 attr
->ah_attr
.grh
.traffic_class
= cmd
.dest
.traffic_class
;
1338 attr
->ah_attr
.dlid
= cmd
.dest
.dlid
;
1339 attr
->ah_attr
.sl
= cmd
.dest
.sl
;
1340 attr
->ah_attr
.src_path_bits
= cmd
.dest
.src_path_bits
;
1341 attr
->ah_attr
.static_rate
= cmd
.dest
.static_rate
;
1342 attr
->ah_attr
.ah_flags
= cmd
.dest
.is_global
? IB_AH_GRH
: 0;
1343 attr
->ah_attr
.port_num
= cmd
.dest
.port_num
;
1345 memcpy(attr
->alt_ah_attr
.grh
.dgid
.raw
, cmd
.alt_dest
.dgid
, 16);
1346 attr
->alt_ah_attr
.grh
.flow_label
= cmd
.alt_dest
.flow_label
;
1347 attr
->alt_ah_attr
.grh
.sgid_index
= cmd
.alt_dest
.sgid_index
;
1348 attr
->alt_ah_attr
.grh
.hop_limit
= cmd
.alt_dest
.hop_limit
;
1349 attr
->alt_ah_attr
.grh
.traffic_class
= cmd
.alt_dest
.traffic_class
;
1350 attr
->alt_ah_attr
.dlid
= cmd
.alt_dest
.dlid
;
1351 attr
->alt_ah_attr
.sl
= cmd
.alt_dest
.sl
;
1352 attr
->alt_ah_attr
.src_path_bits
= cmd
.alt_dest
.src_path_bits
;
1353 attr
->alt_ah_attr
.static_rate
= cmd
.alt_dest
.static_rate
;
1354 attr
->alt_ah_attr
.ah_flags
= cmd
.alt_dest
.is_global
? IB_AH_GRH
: 0;
1355 attr
->alt_ah_attr
.port_num
= cmd
.alt_dest
.port_num
;
1357 ret
= qp
->device
->modify_qp(qp
, attr
, cmd
.attr_mask
, &udata
);
1372 ssize_t
ib_uverbs_destroy_qp(struct ib_uverbs_file
*file
,
1373 const char __user
*buf
, int in_len
,
1376 struct ib_uverbs_destroy_qp cmd
;
1377 struct ib_uverbs_destroy_qp_resp resp
;
1378 struct ib_uobject
*uobj
;
1380 struct ib_uqp_object
*obj
;
1383 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1386 memset(&resp
, 0, sizeof resp
);
1388 uobj
= idr_write_uobj(&ib_uverbs_qp_idr
, cmd
.qp_handle
, file
->ucontext
);
1392 obj
= container_of(uobj
, struct ib_uqp_object
, uevent
.uobject
);
1394 if (!list_empty(&obj
->mcast_list
)) {
1395 put_uobj_write(uobj
);
1399 ret
= ib_destroy_qp(qp
);
1403 put_uobj_write(uobj
);
1408 idr_remove_uobj(&ib_uverbs_qp_idr
, uobj
);
1410 mutex_lock(&file
->mutex
);
1411 list_del(&uobj
->list
);
1412 mutex_unlock(&file
->mutex
);
1414 ib_uverbs_release_uevent(file
, &obj
->uevent
);
1416 resp
.events_reported
= obj
->uevent
.events_reported
;
1420 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1421 &resp
, sizeof resp
))
1427 ssize_t
ib_uverbs_post_send(struct ib_uverbs_file
*file
,
1428 const char __user
*buf
, int in_len
,
1431 struct ib_uverbs_post_send cmd
;
1432 struct ib_uverbs_post_send_resp resp
;
1433 struct ib_uverbs_send_wr
*user_wr
;
1434 struct ib_send_wr
*wr
= NULL
, *last
, *next
, *bad_wr
;
1438 ssize_t ret
= -EINVAL
;
1440 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1443 if (in_len
< sizeof cmd
+ cmd
.wqe_size
* cmd
.wr_count
+
1444 cmd
.sge_count
* sizeof (struct ib_uverbs_sge
))
1447 if (cmd
.wqe_size
< sizeof (struct ib_uverbs_send_wr
))
1450 user_wr
= kmalloc(cmd
.wqe_size
, GFP_KERNEL
);
1454 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1458 is_ud
= qp
->qp_type
== IB_QPT_UD
;
1461 for (i
= 0; i
< cmd
.wr_count
; ++i
) {
1462 if (copy_from_user(user_wr
,
1463 buf
+ sizeof cmd
+ i
* cmd
.wqe_size
,
1469 if (user_wr
->num_sge
+ sg_ind
> cmd
.sge_count
) {
1474 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
1475 user_wr
->num_sge
* sizeof (struct ib_sge
),
1489 next
->wr_id
= user_wr
->wr_id
;
1490 next
->num_sge
= user_wr
->num_sge
;
1491 next
->opcode
= user_wr
->opcode
;
1492 next
->send_flags
= user_wr
->send_flags
;
1495 next
->wr
.ud
.ah
= idr_read_ah(user_wr
->wr
.ud
.ah
,
1497 if (!next
->wr
.ud
.ah
) {
1501 next
->wr
.ud
.remote_qpn
= user_wr
->wr
.ud
.remote_qpn
;
1502 next
->wr
.ud
.remote_qkey
= user_wr
->wr
.ud
.remote_qkey
;
1504 switch (next
->opcode
) {
1505 case IB_WR_RDMA_WRITE_WITH_IMM
:
1507 (__be32 __force
) user_wr
->ex
.imm_data
;
1508 case IB_WR_RDMA_WRITE
:
1509 case IB_WR_RDMA_READ
:
1510 next
->wr
.rdma
.remote_addr
=
1511 user_wr
->wr
.rdma
.remote_addr
;
1512 next
->wr
.rdma
.rkey
=
1513 user_wr
->wr
.rdma
.rkey
;
1515 case IB_WR_SEND_WITH_IMM
:
1517 (__be32 __force
) user_wr
->ex
.imm_data
;
1519 case IB_WR_SEND_WITH_INV
:
1520 next
->ex
.invalidate_rkey
=
1521 user_wr
->ex
.invalidate_rkey
;
1523 case IB_WR_ATOMIC_CMP_AND_SWP
:
1524 case IB_WR_ATOMIC_FETCH_AND_ADD
:
1525 next
->wr
.atomic
.remote_addr
=
1526 user_wr
->wr
.atomic
.remote_addr
;
1527 next
->wr
.atomic
.compare_add
=
1528 user_wr
->wr
.atomic
.compare_add
;
1529 next
->wr
.atomic
.swap
= user_wr
->wr
.atomic
.swap
;
1530 next
->wr
.atomic
.rkey
= user_wr
->wr
.atomic
.rkey
;
1537 if (next
->num_sge
) {
1538 next
->sg_list
= (void *) next
+
1539 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
1540 if (copy_from_user(next
->sg_list
,
1542 cmd
.wr_count
* cmd
.wqe_size
+
1543 sg_ind
* sizeof (struct ib_sge
),
1544 next
->num_sge
* sizeof (struct ib_sge
))) {
1548 sg_ind
+= next
->num_sge
;
1550 next
->sg_list
= NULL
;
1554 ret
= qp
->device
->post_send(qp
, wr
, &bad_wr
);
1556 for (next
= wr
; next
; next
= next
->next
) {
1562 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1563 &resp
, sizeof resp
))
1570 if (is_ud
&& wr
->wr
.ud
.ah
)
1571 put_ah_read(wr
->wr
.ud
.ah
);
1580 return ret
? ret
: in_len
;
1583 static struct ib_recv_wr
*ib_uverbs_unmarshall_recv(const char __user
*buf
,
1589 struct ib_uverbs_recv_wr
*user_wr
;
1590 struct ib_recv_wr
*wr
= NULL
, *last
, *next
;
1595 if (in_len
< wqe_size
* wr_count
+
1596 sge_count
* sizeof (struct ib_uverbs_sge
))
1597 return ERR_PTR(-EINVAL
);
1599 if (wqe_size
< sizeof (struct ib_uverbs_recv_wr
))
1600 return ERR_PTR(-EINVAL
);
1602 user_wr
= kmalloc(wqe_size
, GFP_KERNEL
);
1604 return ERR_PTR(-ENOMEM
);
1608 for (i
= 0; i
< wr_count
; ++i
) {
1609 if (copy_from_user(user_wr
, buf
+ i
* wqe_size
,
1615 if (user_wr
->num_sge
+ sg_ind
> sge_count
) {
1620 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
1621 user_wr
->num_sge
* sizeof (struct ib_sge
),
1635 next
->wr_id
= user_wr
->wr_id
;
1636 next
->num_sge
= user_wr
->num_sge
;
1638 if (next
->num_sge
) {
1639 next
->sg_list
= (void *) next
+
1640 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
1641 if (copy_from_user(next
->sg_list
,
1642 buf
+ wr_count
* wqe_size
+
1643 sg_ind
* sizeof (struct ib_sge
),
1644 next
->num_sge
* sizeof (struct ib_sge
))) {
1648 sg_ind
+= next
->num_sge
;
1650 next
->sg_list
= NULL
;
1665 return ERR_PTR(ret
);
1668 ssize_t
ib_uverbs_post_recv(struct ib_uverbs_file
*file
,
1669 const char __user
*buf
, int in_len
,
1672 struct ib_uverbs_post_recv cmd
;
1673 struct ib_uverbs_post_recv_resp resp
;
1674 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
1676 ssize_t ret
= -EINVAL
;
1678 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1681 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
1682 in_len
- sizeof cmd
, cmd
.wr_count
,
1683 cmd
.sge_count
, cmd
.wqe_size
);
1687 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1692 ret
= qp
->device
->post_recv(qp
, wr
, &bad_wr
);
1697 for (next
= wr
; next
; next
= next
->next
) {
1703 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1704 &resp
, sizeof resp
))
1714 return ret
? ret
: in_len
;
1717 ssize_t
ib_uverbs_post_srq_recv(struct ib_uverbs_file
*file
,
1718 const char __user
*buf
, int in_len
,
1721 struct ib_uverbs_post_srq_recv cmd
;
1722 struct ib_uverbs_post_srq_recv_resp resp
;
1723 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
1725 ssize_t ret
= -EINVAL
;
1727 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1730 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
1731 in_len
- sizeof cmd
, cmd
.wr_count
,
1732 cmd
.sge_count
, cmd
.wqe_size
);
1736 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
1741 ret
= srq
->device
->post_srq_recv(srq
, wr
, &bad_wr
);
1746 for (next
= wr
; next
; next
= next
->next
) {
1752 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1753 &resp
, sizeof resp
))
1763 return ret
? ret
: in_len
;
1766 ssize_t
ib_uverbs_create_ah(struct ib_uverbs_file
*file
,
1767 const char __user
*buf
, int in_len
,
1770 struct ib_uverbs_create_ah cmd
;
1771 struct ib_uverbs_create_ah_resp resp
;
1772 struct ib_uobject
*uobj
;
1775 struct ib_ah_attr attr
;
1778 if (out_len
< sizeof resp
)
1781 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1784 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
1788 init_uobj(uobj
, cmd
.user_handle
, file
->ucontext
, &ah_lock_key
);
1789 down_write(&uobj
->mutex
);
1791 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1797 attr
.dlid
= cmd
.attr
.dlid
;
1798 attr
.sl
= cmd
.attr
.sl
;
1799 attr
.src_path_bits
= cmd
.attr
.src_path_bits
;
1800 attr
.static_rate
= cmd
.attr
.static_rate
;
1801 attr
.ah_flags
= cmd
.attr
.is_global
? IB_AH_GRH
: 0;
1802 attr
.port_num
= cmd
.attr
.port_num
;
1803 attr
.grh
.flow_label
= cmd
.attr
.grh
.flow_label
;
1804 attr
.grh
.sgid_index
= cmd
.attr
.grh
.sgid_index
;
1805 attr
.grh
.hop_limit
= cmd
.attr
.grh
.hop_limit
;
1806 attr
.grh
.traffic_class
= cmd
.attr
.grh
.traffic_class
;
1807 memcpy(attr
.grh
.dgid
.raw
, cmd
.attr
.grh
.dgid
, 16);
1809 ah
= ib_create_ah(pd
, &attr
);
1818 ret
= idr_add_uobj(&ib_uverbs_ah_idr
, uobj
);
1822 resp
.ah_handle
= uobj
->id
;
1824 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1825 &resp
, sizeof resp
)) {
1832 mutex_lock(&file
->mutex
);
1833 list_add_tail(&uobj
->list
, &file
->ucontext
->ah_list
);
1834 mutex_unlock(&file
->mutex
);
1838 up_write(&uobj
->mutex
);
1843 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
1852 put_uobj_write(uobj
);
1856 ssize_t
ib_uverbs_destroy_ah(struct ib_uverbs_file
*file
,
1857 const char __user
*buf
, int in_len
, int out_len
)
1859 struct ib_uverbs_destroy_ah cmd
;
1861 struct ib_uobject
*uobj
;
1864 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1867 uobj
= idr_write_uobj(&ib_uverbs_ah_idr
, cmd
.ah_handle
, file
->ucontext
);
1872 ret
= ib_destroy_ah(ah
);
1876 put_uobj_write(uobj
);
1881 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
1883 mutex_lock(&file
->mutex
);
1884 list_del(&uobj
->list
);
1885 mutex_unlock(&file
->mutex
);
1892 ssize_t
ib_uverbs_attach_mcast(struct ib_uverbs_file
*file
,
1893 const char __user
*buf
, int in_len
,
1896 struct ib_uverbs_attach_mcast cmd
;
1898 struct ib_uqp_object
*obj
;
1899 struct ib_uverbs_mcast_entry
*mcast
;
1902 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1905 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1909 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
1911 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
1912 if (cmd
.mlid
== mcast
->lid
&&
1913 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
1918 mcast
= kmalloc(sizeof *mcast
, GFP_KERNEL
);
1924 mcast
->lid
= cmd
.mlid
;
1925 memcpy(mcast
->gid
.raw
, cmd
.gid
, sizeof mcast
->gid
.raw
);
1927 ret
= ib_attach_mcast(qp
, &mcast
->gid
, cmd
.mlid
);
1929 list_add_tail(&mcast
->list
, &obj
->mcast_list
);
1936 return ret
? ret
: in_len
;
1939 ssize_t
ib_uverbs_detach_mcast(struct ib_uverbs_file
*file
,
1940 const char __user
*buf
, int in_len
,
1943 struct ib_uverbs_detach_mcast cmd
;
1944 struct ib_uqp_object
*obj
;
1946 struct ib_uverbs_mcast_entry
*mcast
;
1949 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1952 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1956 ret
= ib_detach_mcast(qp
, (union ib_gid
*) cmd
.gid
, cmd
.mlid
);
1960 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
1962 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
1963 if (cmd
.mlid
== mcast
->lid
&&
1964 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
1965 list_del(&mcast
->list
);
1973 return ret
? ret
: in_len
;
1976 ssize_t
ib_uverbs_create_srq(struct ib_uverbs_file
*file
,
1977 const char __user
*buf
, int in_len
,
1980 struct ib_uverbs_create_srq cmd
;
1981 struct ib_uverbs_create_srq_resp resp
;
1982 struct ib_udata udata
;
1983 struct ib_uevent_object
*obj
;
1986 struct ib_srq_init_attr attr
;
1989 if (out_len
< sizeof resp
)
1992 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1995 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1996 (unsigned long) cmd
.response
+ sizeof resp
,
1997 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1999 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
2003 init_uobj(&obj
->uobject
, cmd
.user_handle
, file
->ucontext
, &srq_lock_key
);
2004 down_write(&obj
->uobject
.mutex
);
2006 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
2012 attr
.event_handler
= ib_uverbs_srq_event_handler
;
2013 attr
.srq_context
= file
;
2014 attr
.attr
.max_wr
= cmd
.max_wr
;
2015 attr
.attr
.max_sge
= cmd
.max_sge
;
2016 attr
.attr
.srq_limit
= cmd
.srq_limit
;
2018 obj
->events_reported
= 0;
2019 INIT_LIST_HEAD(&obj
->event_list
);
2021 srq
= pd
->device
->create_srq(pd
, &attr
, &udata
);
2027 srq
->device
= pd
->device
;
2029 srq
->uobject
= &obj
->uobject
;
2030 srq
->event_handler
= attr
.event_handler
;
2031 srq
->srq_context
= attr
.srq_context
;
2032 atomic_inc(&pd
->usecnt
);
2033 atomic_set(&srq
->usecnt
, 0);
2035 obj
->uobject
.object
= srq
;
2036 ret
= idr_add_uobj(&ib_uverbs_srq_idr
, &obj
->uobject
);
2040 memset(&resp
, 0, sizeof resp
);
2041 resp
.srq_handle
= obj
->uobject
.id
;
2042 resp
.max_wr
= attr
.attr
.max_wr
;
2043 resp
.max_sge
= attr
.attr
.max_sge
;
2045 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2046 &resp
, sizeof resp
)) {
2053 mutex_lock(&file
->mutex
);
2054 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->srq_list
);
2055 mutex_unlock(&file
->mutex
);
2057 obj
->uobject
.live
= 1;
2059 up_write(&obj
->uobject
.mutex
);
2064 idr_remove_uobj(&ib_uverbs_srq_idr
, &obj
->uobject
);
2067 ib_destroy_srq(srq
);
2073 put_uobj_write(&obj
->uobject
);
2077 ssize_t
ib_uverbs_modify_srq(struct ib_uverbs_file
*file
,
2078 const char __user
*buf
, int in_len
,
2081 struct ib_uverbs_modify_srq cmd
;
2082 struct ib_udata udata
;
2084 struct ib_srq_attr attr
;
2087 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2090 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
2093 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
2097 attr
.max_wr
= cmd
.max_wr
;
2098 attr
.srq_limit
= cmd
.srq_limit
;
2100 ret
= srq
->device
->modify_srq(srq
, &attr
, cmd
.attr_mask
, &udata
);
2104 return ret
? ret
: in_len
;
2107 ssize_t
ib_uverbs_query_srq(struct ib_uverbs_file
*file
,
2108 const char __user
*buf
,
2109 int in_len
, int out_len
)
2111 struct ib_uverbs_query_srq cmd
;
2112 struct ib_uverbs_query_srq_resp resp
;
2113 struct ib_srq_attr attr
;
2117 if (out_len
< sizeof resp
)
2120 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2123 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
2127 ret
= ib_query_srq(srq
, &attr
);
2134 memset(&resp
, 0, sizeof resp
);
2136 resp
.max_wr
= attr
.max_wr
;
2137 resp
.max_sge
= attr
.max_sge
;
2138 resp
.srq_limit
= attr
.srq_limit
;
2140 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2141 &resp
, sizeof resp
))
2147 ssize_t
ib_uverbs_destroy_srq(struct ib_uverbs_file
*file
,
2148 const char __user
*buf
, int in_len
,
2151 struct ib_uverbs_destroy_srq cmd
;
2152 struct ib_uverbs_destroy_srq_resp resp
;
2153 struct ib_uobject
*uobj
;
2155 struct ib_uevent_object
*obj
;
2158 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2161 uobj
= idr_write_uobj(&ib_uverbs_srq_idr
, cmd
.srq_handle
, file
->ucontext
);
2165 obj
= container_of(uobj
, struct ib_uevent_object
, uobject
);
2167 ret
= ib_destroy_srq(srq
);
2171 put_uobj_write(uobj
);
2176 idr_remove_uobj(&ib_uverbs_srq_idr
, uobj
);
2178 mutex_lock(&file
->mutex
);
2179 list_del(&uobj
->list
);
2180 mutex_unlock(&file
->mutex
);
2182 ib_uverbs_release_uevent(file
, obj
);
2184 memset(&resp
, 0, sizeof resp
);
2185 resp
.events_reported
= obj
->events_reported
;
2189 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2190 &resp
, sizeof resp
))
2193 return ret
? ret
: in_len
;