4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
29 * Tavor Resource Management Routines
31 * Implements all the routines necessary for setup, teardown, and
32 * alloc/free of all Tavor resources, including those that are managed
33 * by Tavor hardware or which live in Tavor's direct attached DDR memory.
36 #include <sys/sysmacros.h>
37 #include <sys/types.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
43 #include <sys/bitmap.h>
45 #include <sys/ib/adapters/tavor/tavor.h>
48 * The following routines are used for initializing and destroying
49 * the resource pools used by the Tavor resource allocation routines.
50 * They consist of four classes of object:
52 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor
53 * command interface routines. Mailboxes are used to pass information
54 * back and forth to the Tavor firmware. Either type of mailbox may
55 * be allocated from Tavor's direct attached DDR memory or from system
56 * memory (although currently all "In" mailboxes are in DDR and all "out"
57 * mailboxes come from system memory.
59 * HW entry objects: These objects represent resources required by the Tavor
60 * hardware. These objects include things like Queue Pair contexts (QPC),
61 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
62 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
63 * Protection Table entries (MPT), Memory Translation Table entries (MTT).
65 * What these objects all have in common is that they are each required
66 * to come from DDR memory, they are always allocated from tables, and
67 * they are not to be directly accessed (read or written) by driver
69 * One notable exceptions to this rule are the Extended QP contexts (EQPC),
70 * and the UAR scratch area (UAR_SCR), both of which are not directly
71 * accessible through the Tavor resource allocation routines, but both
72 * of which are also required to reside in DDR memory and are not to be
73 * manipulated by driver software (they are privately managed by Tavor
75 * The other notable exceptions are the UAR pages (UAR_PG) which are
76 * allocated from the UAR address space rather than DDR, and the UD
77 * address vectors (UDAV) which are similar to the common object types
78 * with the major difference being that UDAVs _are_ directly read and
79 * written by driver software.
81 * SW handle objects: These objects represent resources required by Tavor
82 * driver software. They are primarily software tracking structures,
83 * which are allocated from system memory (using kmem_cache). Several of
84 * the objects have both a "constructor" and "destructor" method
85 * associated with them (see below).
87 * Protection Domain (PD) handle objects: These objects are very much like
88 * a SW handle object with the notable difference that all PD handle
89 * objects have an actual Protection Domain number (PD) associated with
90 * them (and the PD number is allocated/managed through a separate
91 * vmem_arena specifically set aside for this purpose.
94 static int tavor_rsrc_mbox_init(tavor_state_t
*state
,
95 tavor_rsrc_mbox_info_t
*info
);
96 static void tavor_rsrc_mbox_fini(tavor_state_t
*state
,
97 tavor_rsrc_mbox_info_t
*info
);
99 static int tavor_rsrc_hw_entries_init(tavor_state_t
*state
,
100 tavor_rsrc_hw_entry_info_t
*info
);
101 static void tavor_rsrc_hw_entries_fini(tavor_state_t
*state
,
102 tavor_rsrc_hw_entry_info_t
*info
);
104 static int tavor_rsrc_sw_handles_init(tavor_state_t
*state
,
105 tavor_rsrc_sw_hdl_info_t
*info
);
106 static void tavor_rsrc_sw_handles_fini(tavor_state_t
*state
,
107 tavor_rsrc_sw_hdl_info_t
*info
);
109 static int tavor_rsrc_pd_handles_init(tavor_state_t
*state
,
110 tavor_rsrc_sw_hdl_info_t
*info
);
111 static void tavor_rsrc_pd_handles_fini(tavor_state_t
*state
,
112 tavor_rsrc_sw_hdl_info_t
*info
);
115 * The following routines are used for allocating and freeing the specific
116 * types of objects described above from their associated resource pools.
118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t
*pool_info
,
119 uint_t num
, tavor_rsrc_t
*hdl
);
120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t
*pool_info
,
123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t
*pool_info
,
124 uint_t num
, uint_t num_align
, ddi_acc_handle_t acc_handle
,
125 uint_t sleepflag
, tavor_rsrc_t
*hdl
);
126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t
*pool_info
,
129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t
*pool_info
,
130 uint_t sleepflag
, tavor_rsrc_t
*hdl
);
131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t
*pool_info
,
134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t
*pool_info
,
135 uint_t sleepflag
, tavor_rsrc_t
*hdl
);
136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t
*pool_info
,
140 * The following routines are the constructors and destructors for several
141 * of the SW handle type objects. For certain types of SW handles objects
142 * (all of which are implemented using kmem_cache), we need to do some
143 * special field initialization (specifically, mutex_init/destroy). These
144 * routines enable that init and teardown.
146 static int tavor_rsrc_pdhdl_constructor(void *pd
, void *priv
, int flags
);
147 static void tavor_rsrc_pdhdl_destructor(void *pd
, void *state
);
148 static int tavor_rsrc_cqhdl_constructor(void *cq
, void *priv
, int flags
);
149 static void tavor_rsrc_cqhdl_destructor(void *cq
, void *state
);
150 static int tavor_rsrc_qphdl_constructor(void *cq
, void *priv
, int flags
);
151 static void tavor_rsrc_qphdl_destructor(void *cq
, void *state
);
152 static int tavor_rsrc_srqhdl_constructor(void *srq
, void *priv
, int flags
);
153 static void tavor_rsrc_srqhdl_destructor(void *srq
, void *state
);
154 static int tavor_rsrc_refcnt_constructor(void *rc
, void *priv
, int flags
);
155 static void tavor_rsrc_refcnt_destructor(void *rc
, void *state
);
156 static int tavor_rsrc_ahhdl_constructor(void *ah
, void *priv
, int flags
);
157 static void tavor_rsrc_ahhdl_destructor(void *ah
, void *state
);
158 static int tavor_rsrc_mrhdl_constructor(void *mr
, void *priv
, int flags
);
159 static void tavor_rsrc_mrhdl_destructor(void *mr
, void *state
);
162 * Special routine to calculate and return the size of a MCG object based
163 * on current driver configuration (specifically, the number of QP per MCG
164 * that has been configured.
166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t
*state
,
167 uint_t
*mcg_size_shift
);
173 * Context: Can be called from interrupt or base context.
174 * The "sleepflag" parameter is used by all object allocators to
175 * determine whether to SLEEP for resources or not.
178 tavor_rsrc_alloc(tavor_state_t
*state
, tavor_rsrc_type_t rsrc
, uint_t num
,
179 uint_t sleepflag
, tavor_rsrc_t
**hdl
)
181 tavor_rsrc_pool_info_t
*rsrc_pool
;
182 tavor_rsrc_t
*tmp_rsrc_hdl
;
183 int flag
, status
= DDI_FAILURE
;
185 TAVOR_TNF_ENTER(tavor_rsrc_alloc
);
187 ASSERT(state
!= NULL
);
190 rsrc_pool
= &state
->ts_rsrc_hdl
[rsrc
];
191 ASSERT(rsrc_pool
!= NULL
);
194 * Allocate space for the object used to track the resource handle
196 flag
= (sleepflag
== TAVOR_SLEEP
) ? KM_SLEEP
: KM_NOSLEEP
;
197 tmp_rsrc_hdl
= (tavor_rsrc_t
*)kmem_cache_alloc(state
->ts_rsrc_cache
,
199 if (tmp_rsrc_hdl
== NULL
) {
200 TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail
, TAVOR_TNF_ERROR
, "");
201 TAVOR_TNF_EXIT(tavor_rsrc_alloc
);
202 return (DDI_FAILURE
);
204 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl
))
207 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call
208 * to know what type of resource is being freed.
210 tmp_rsrc_hdl
->rsrc_type
= rsrc
;
213 * Depending on resource type, call the appropriate alloc routine
215 switch (rsrc_pool
->rsrc_type
) {
218 case TAVOR_INTR_IN_MBOX
:
219 case TAVOR_INTR_OUT_MBOX
:
220 status
= tavor_rsrc_mbox_alloc(rsrc_pool
, num
, tmp_rsrc_hdl
);
229 * Because these objects are NOT accessed by Tavor driver
230 * software, we set the acc_handle parameter to zero. But
231 * if they are allocated in multiples, we specify here that
232 * they must be aligned on a more restrictive boundary.
234 status
= tavor_rsrc_hw_entry_alloc(rsrc_pool
, num
, num
, 0,
235 sleepflag
, tmp_rsrc_hdl
);
240 * Because these MPT objects are sometimes accessed by Tavor
241 * driver software (FMR), we set the acc_handle parameter. But
242 * if they are allocated in multiples, we specify here that
243 * they must be aligned on a more restrictive boundary.
245 status
= tavor_rsrc_hw_entry_alloc(rsrc_pool
, num
, num
,
246 state
->ts_reg_ddrhdl
, sleepflag
, tmp_rsrc_hdl
);
251 * Tavor MCG entries are also NOT accessed by Tavor driver
252 * software, but because MCG entries do not have the same
253 * alignnment restrictions we loosen the constraint here.
255 status
= tavor_rsrc_hw_entry_alloc(rsrc_pool
, num
, 1, 0,
256 sleepflag
, tmp_rsrc_hdl
);
262 * Because MTT segments are among the few HW resources that
263 * may be allocated in odd numbers, we specify a less
264 * restrictive alignment than for the above resources.
266 * Also because both UDAV and MTT segment objects are read
267 * and/or written by Tavor driver software, we set the
268 * acc_handle parameter to point to the ddi_acc_handle_t for
269 * the Tavor DDR memory.
271 status
= tavor_rsrc_hw_entry_alloc(rsrc_pool
, num
, 1,
272 state
->ts_reg_ddrhdl
, sleepflag
, tmp_rsrc_hdl
);
277 * Because UAR pages are written by Tavor driver software (for
278 * doorbells), we set the acc_handle parameter to point to
279 * the ddi_acc_handle_t for the Tavor UAR memory.
281 status
= tavor_rsrc_hw_entry_alloc(rsrc_pool
, num
, 1,
282 state
->ts_reg_uarhdl
, sleepflag
, tmp_rsrc_hdl
);
292 status
= tavor_rsrc_swhdl_alloc(rsrc_pool
, sleepflag
,
297 status
= tavor_rsrc_pdhdl_alloc(rsrc_pool
, sleepflag
,
302 TAVOR_WARNING(state
, "unexpected resource type in alloc");
303 TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail
,
304 TAVOR_TNF_ERROR
, "");
309 * If the resource allocation failed, then free the special resource
310 * tracking structure and return failure. Otherwise return the
311 * handle for the resource tracking structure.
313 if (status
!= DDI_SUCCESS
) {
314 kmem_cache_free(state
->ts_rsrc_cache
, tmp_rsrc_hdl
);
316 TNF_PROBE_1(tavor_rsrc_alloc_fail
, TAVOR_TNF_ERROR
, "",
317 tnf_uint
, rsrc_type
, rsrc_pool
->rsrc_type
);
318 TAVOR_TNF_EXIT(tavor_rsrc_alloc
);
319 return (DDI_FAILURE
);
322 TAVOR_TNF_EXIT(tavor_rsrc_alloc
);
323 return (DDI_SUCCESS
);
330 * Context: Can be called from interrupt or base context.
333 tavor_rsrc_free(tavor_state_t
*state
, tavor_rsrc_t
**hdl
)
335 tavor_rsrc_pool_info_t
*rsrc_pool
;
337 TAVOR_TNF_ENTER(tavor_rsrc_free
);
339 ASSERT(state
!= NULL
);
342 rsrc_pool
= &state
->ts_rsrc_hdl
[(*hdl
)->rsrc_type
];
343 ASSERT(rsrc_pool
!= NULL
);
346 * Depending on resource type, call the appropriate free routine
348 switch (rsrc_pool
->rsrc_type
) {
351 case TAVOR_INTR_IN_MBOX
:
352 case TAVOR_INTR_OUT_MBOX
:
353 tavor_rsrc_mbox_free(rsrc_pool
, *hdl
);
366 tavor_rsrc_hw_entry_free(rsrc_pool
, *hdl
);
376 tavor_rsrc_swhdl_free(rsrc_pool
, *hdl
);
380 tavor_rsrc_pdhdl_free(rsrc_pool
, *hdl
);
384 TAVOR_WARNING(state
, "unexpected resource type in free");
385 TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail
,
386 TAVOR_TNF_ERROR
, "");
391 * Free the special resource tracking structure, set the handle to
394 kmem_cache_free(state
->ts_rsrc_cache
, *hdl
);
397 TAVOR_TNF_EXIT(tavor_rsrc_free
);
402 * tavor_rsrc_init_phase1()
404 * Completes the first phase of Tavor resource/configuration init.
405 * This involves creating the kmem_cache for the "tavor_rsrc_t"
406 * structs, allocating the space for the resource pool handles,
407 * and setting up the "Out" mailboxes.
409 * When this function completes, the Tavor driver is ready to
410 * post the following commands which return information only in the
411 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
412 * If any of these commands are to be posted at this time, they must be
413 * done so only when "spinning" (as the outstanding command list and
414 * EQ setup code has not yet run)
416 * Context: Only called from attach() path context
419 tavor_rsrc_init_phase1(tavor_state_t
*state
)
421 tavor_rsrc_pool_info_t
*rsrc_pool
;
422 tavor_rsrc_mbox_info_t mbox_info
;
423 tavor_rsrc_cleanup_level_t cleanup
;
424 tavor_cfg_profile_t
*cfgprof
;
427 char *errormsg
, *rsrc_name
;
429 TAVOR_TNF_ENTER(tavor_rsrc_init_phase1
);
431 ASSERT(state
!= NULL
);
433 /* This is where Phase 1 of resource initialization begins */
434 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL0
;
436 /* Build kmem cache name from Tavor instance */
437 rsrc_name
= (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN
, KM_SLEEP
);
438 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_RSRC_CACHE
);
441 * Create the kmem_cache for "tavor_rsrc_t" structures
442 * (kmem_cache_create will SLEEP until successful)
444 state
->ts_rsrc_cache
= kmem_cache_create(rsrc_name
,
445 sizeof (tavor_rsrc_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
448 * Allocate an array of tavor_rsrc_pool_info_t's (used in all
449 * subsequent resource allocations)
451 state
->ts_rsrc_hdl
= kmem_zalloc(TAVOR_NUM_RESOURCES
*
452 sizeof (tavor_rsrc_pool_info_t
), KM_SLEEP
);
454 cfgprof
= state
->ts_cfg_profile
;
457 * Initialize the resource pool for "Out" mailboxes. Notice that
458 * the number of "Out" mailboxes, their size, and their location
459 * (DDR or system memory) is configurable. By default, however,
460 * all "Out" mailboxes are located in system memory only (because
461 * they are primarily read from and never written to)
463 num
= ((uint64_t)1 << cfgprof
->cp_log_num_outmbox
);
464 size
= ((uint64_t)1 << cfgprof
->cp_log_outmbox_size
);
465 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_OUT_MBOX
];
466 rsrc_pool
->rsrc_type
= TAVOR_OUT_MBOX
;
467 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
468 rsrc_pool
->rsrc_pool_size
= (size
* num
);
469 rsrc_pool
->rsrc_shift
= cfgprof
->cp_log_outmbox_size
;
470 rsrc_pool
->rsrc_quantum
= size
;
471 rsrc_pool
->rsrc_align
= TAVOR_MBOX_ALIGN
;
472 rsrc_pool
->rsrc_state
= state
;
473 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_OUTMBOX_VMEM
);
474 mbox_info
.mbi_num
= num
;
475 mbox_info
.mbi_size
= size
;
476 mbox_info
.mbi_rsrcpool
= rsrc_pool
;
477 mbox_info
.mbi_rsrcname
= rsrc_name
;
478 status
= tavor_rsrc_mbox_init(state
, &mbox_info
);
479 if (status
!= DDI_SUCCESS
) {
480 tavor_rsrc_fini(state
, cleanup
);
481 /* Set "status" and "errormsg" and goto failure */
482 TAVOR_TNF_FAIL(DDI_FAILURE
, "out mailboxes");
483 goto rsrcinitp1_fail
;
485 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL1
;
488 * Initialize the Tavor "Out" mailbox list. This step actually uses
489 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
490 * "Out" mailboxes, bind them for DMA access, and arrange them into
491 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
494 status
= tavor_outmbox_list_init(state
);
495 if (status
!= DDI_SUCCESS
) {
496 tavor_rsrc_fini(state
, cleanup
);
497 /* Set "status" and "errormsg" and goto failure */
498 TAVOR_TNF_FAIL(DDI_FAILURE
, "out mailbox list");
499 goto rsrcinitp1_fail
;
501 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL2
;
504 * Initialize the resource pool for interrupt "Out" mailboxes. Notice
505 * that the number of interrupt "Out" mailboxes, their size, and their
506 * location (DDR or system memory) is configurable. By default,
507 * however, all interrupt "Out" mailboxes are located in system memory
508 * only (because they are primarily read from and never written to)
510 num
= ((uint64_t)1 << cfgprof
->cp_log_num_intr_outmbox
);
511 size
= ((uint64_t)1 << cfgprof
->cp_log_outmbox_size
);
512 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_INTR_OUT_MBOX
];
513 rsrc_pool
->rsrc_type
= TAVOR_INTR_OUT_MBOX
;
514 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
515 rsrc_pool
->rsrc_pool_size
= (size
* num
);
516 rsrc_pool
->rsrc_shift
= cfgprof
->cp_log_outmbox_size
;
517 rsrc_pool
->rsrc_quantum
= size
;
518 rsrc_pool
->rsrc_align
= TAVOR_MBOX_ALIGN
;
519 rsrc_pool
->rsrc_state
= state
;
520 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_INTR_OUTMBOX_VMEM
);
521 mbox_info
.mbi_num
= num
;
522 mbox_info
.mbi_size
= size
;
523 mbox_info
.mbi_rsrcpool
= rsrc_pool
;
524 mbox_info
.mbi_rsrcname
= rsrc_name
;
525 status
= tavor_rsrc_mbox_init(state
, &mbox_info
);
526 if (status
!= DDI_SUCCESS
) {
527 tavor_rsrc_fini(state
, cleanup
);
528 /* Set "status" and "errormsg" and goto failure */
529 TAVOR_TNF_FAIL(DDI_FAILURE
, "out intr mailboxes");
530 goto rsrcinitp1_fail
;
532 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL3
;
535 * Initialize the Tavor "Out" mailbox list. This step actually uses
536 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
537 * "Out" mailboxes, bind them for DMA access, and arrange them into
538 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
541 status
= tavor_intr_outmbox_list_init(state
);
542 if (status
!= DDI_SUCCESS
) {
543 tavor_rsrc_fini(state
, cleanup
);
544 /* Set "status" and "errormsg" and goto failure */
545 TAVOR_TNF_FAIL(DDI_FAILURE
, "out intr mailbox list");
546 goto rsrcinitp1_fail
;
548 cleanup
= TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE
;
550 kmem_free(rsrc_name
, TAVOR_RSRC_NAME_MAXLEN
);
551 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1
);
552 return (DDI_SUCCESS
);
555 kmem_free(rsrc_name
, TAVOR_RSRC_NAME_MAXLEN
);
556 TNF_PROBE_1(tavor_rsrc_init_phase1_fail
, TAVOR_TNF_ERROR
, "",
557 tnf_string
, msg
, errormsg
);
558 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1
);
564 * tavor_rsrc_init_phase2()
565 * Context: Only called from attach() path context
568 tavor_rsrc_init_phase2(tavor_state_t
*state
)
570 tavor_rsrc_sw_hdl_info_t hdl_info
;
571 tavor_rsrc_hw_entry_info_t entry_info
;
572 tavor_rsrc_mbox_info_t mbox_info
;
573 tavor_rsrc_pool_info_t
*rsrc_pool
;
574 tavor_rsrc_cleanup_level_t cleanup
;
575 tavor_cfg_profile_t
*cfgprof
;
576 uint64_t num
, max
, size
, num_prealloc
;
577 uint64_t ddr_size
, fw_size
;
578 uint_t mcg_size
, mcg_size_shift
;
579 uint_t uarscr_size
, mttsegment_sz
;
581 char *errormsg
, *rsrc_name
;
583 TAVOR_TNF_ENTER(tavor_rsrc_init_phase2
);
585 ASSERT(state
!= NULL
);
587 /* Phase 2 initialization begins where Phase 1 left off */
588 cleanup
= TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE
;
591 * Calculate the extent of the DDR size and portion of which that
592 * is already reserved for Tavor firmware. (Note: this information
593 * is available because the QUERY_DDR and QUERY_FW commands have
594 * been posted to Tavor firmware prior to calling this routine)
596 ddr_size
= state
->ts_ddr
.ddr_endaddr
- state
->ts_ddr
.ddr_baseaddr
+ 1;
597 fw_size
= state
->ts_fw
.fw_endaddr
- state
->ts_fw
.fw_baseaddr
+ 1;
599 /* Build the DDR vmem arena name from Tavor instance */
600 rsrc_name
= (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN
, KM_SLEEP
);
601 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_VMEM
);
604 * Do a vmem_create for the entire DDR range (not including the
605 * portion consumed by Tavor firmware). This creates the vmem arena
606 * from which all other DDR objects (specifically, tables of HW
607 * entries) will be allocated.
609 state
->ts_ddrvmem
= vmem_create(rsrc_name
,
610 (void *)(uintptr_t)state
->ts_ddr
.ddr_baseaddr
, (ddr_size
- fw_size
),
611 sizeof (uint64_t), NULL
, NULL
, NULL
, 0, VM_SLEEP
);
612 if (state
->ts_ddrvmem
== NULL
) {
613 tavor_rsrc_fini(state
, cleanup
);
614 /* Set "status" and "errormsg" and goto failure */
615 TAVOR_TNF_FAIL(DDI_FAILURE
, "DDR vmem");
616 goto rsrcinitp2_fail
;
618 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL5
;
621 * Initialize the resource pools for all objects that exist in
622 * Tavor DDR memory. This includes ("In") mailboxes, context tables
623 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
625 cfgprof
= state
->ts_cfg_profile
;
628 * Initialize the resource pool for the MPT table entries. Notice
629 * that the number of MPTs is configurable. The configured value must
630 * be less that the maximum value (obtained from the QUERY_DEV_LIM
631 * command) or the initialization will fail. Note also that a certain
632 * number of MPTs must be set aside for Tavor firmware use.
634 num
= ((uint64_t)1 << cfgprof
->cp_log_num_mpt
);
635 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_mpt
);
636 num_prealloc
= ((uint64_t)1 << state
->ts_devlim
.log_rsvd_mpt
);
637 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_MPT
];
638 rsrc_pool
->rsrc_type
= TAVOR_MPT
;
639 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
640 rsrc_pool
->rsrc_pool_size
= (TAVOR_MPT_SIZE
* num
);
641 rsrc_pool
->rsrc_shift
= TAVOR_MPT_SIZE_SHIFT
;
642 rsrc_pool
->rsrc_quantum
= TAVOR_MPT_SIZE
;
643 rsrc_pool
->rsrc_align
= (TAVOR_MPT_SIZE
* num
);
644 rsrc_pool
->rsrc_state
= state
;
645 rsrc_pool
->rsrc_start
= NULL
;
646 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_MPT_VMEM
);
647 entry_info
.hwi_num
= num
;
648 entry_info
.hwi_max
= max
;
649 entry_info
.hwi_prealloc
= num_prealloc
;
650 entry_info
.hwi_rsrcpool
= rsrc_pool
;
651 entry_info
.hwi_rsrcname
= rsrc_name
;
652 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
653 if (status
!= DDI_SUCCESS
) {
654 tavor_rsrc_fini(state
, cleanup
);
655 /* Set "status" and "errormsg" and goto failure */
656 TAVOR_TNF_FAIL(DDI_FAILURE
, "MPT table");
657 goto rsrcinitp2_fail
;
659 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL6
;
662 * Initialize the resource pool for the MTT table entries. Notice
663 * that the number of MTTs is configurable. The configured value must
664 * be less that the maximum value (obtained from the QUERY_DEV_LIM
665 * command) or the initialization will fail. Note also that a certain
666 * number of MTT segments must be set aside for Tavor firmware use.
668 mttsegment_sz
= (TAVOR_MTTSEG_SIZE
<< TAVOR_MTT_SIZE_SHIFT
);
669 num
= ((uint64_t)1 << cfgprof
->cp_log_num_mttseg
);
670 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_mttseg
);
671 num_prealloc
= ((uint64_t)1 << state
->ts_devlim
.log_rsvd_mttseg
);
672 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_MTT
];
673 rsrc_pool
->rsrc_type
= TAVOR_MTT
;
674 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
675 rsrc_pool
->rsrc_pool_size
= (TAVOR_MTT_SIZE
* num
);
676 rsrc_pool
->rsrc_shift
= TAVOR_MTT_SIZE_SHIFT
;
677 rsrc_pool
->rsrc_quantum
= mttsegment_sz
;
678 rsrc_pool
->rsrc_align
= (TAVOR_MTT_SIZE
* num
);
679 rsrc_pool
->rsrc_state
= state
;
680 rsrc_pool
->rsrc_start
= NULL
;
681 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_MTT_VMEM
);
682 entry_info
.hwi_num
= num
;
683 entry_info
.hwi_max
= max
;
684 entry_info
.hwi_prealloc
= num_prealloc
;
685 entry_info
.hwi_rsrcpool
= rsrc_pool
;
686 entry_info
.hwi_rsrcname
= rsrc_name
;
687 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
688 if (status
!= DDI_SUCCESS
) {
689 tavor_rsrc_fini(state
, cleanup
);
690 /* Set "status" and "errormsg" and goto failure */
691 TAVOR_TNF_FAIL(DDI_FAILURE
, "MTT table");
692 goto rsrcinitp2_fail
;
694 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL7
;
697 * Initialize the resource pool for the QPC table entries. Notice
698 * that the number of QPs is configurable. The configured value must
699 * be less that the maximum value (obtained from the QUERY_DEV_LIM
700 * command) or the initialization will fail. Note also that a certain
701 * number of QP contexts must be set aside for Tavor firmware use.
703 num
= ((uint64_t)1 << cfgprof
->cp_log_num_qp
);
704 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_qp
);
705 num_prealloc
= ((uint64_t)1 << state
->ts_devlim
.log_rsvd_qp
);
706 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_QPC
];
707 rsrc_pool
->rsrc_type
= TAVOR_QPC
;
708 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
709 rsrc_pool
->rsrc_pool_size
= (TAVOR_QPC_SIZE
* num
);
710 rsrc_pool
->rsrc_shift
= TAVOR_QPC_SIZE_SHIFT
;
711 rsrc_pool
->rsrc_quantum
= TAVOR_QPC_SIZE
;
712 rsrc_pool
->rsrc_align
= (TAVOR_QPC_SIZE
* num
);
713 rsrc_pool
->rsrc_state
= state
;
714 rsrc_pool
->rsrc_start
= NULL
;
715 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_QPC_VMEM
);
716 entry_info
.hwi_num
= num
;
717 entry_info
.hwi_max
= max
;
718 entry_info
.hwi_prealloc
= num_prealloc
;
719 entry_info
.hwi_rsrcpool
= rsrc_pool
;
720 entry_info
.hwi_rsrcname
= rsrc_name
;
721 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
722 if (status
!= DDI_SUCCESS
) {
723 tavor_rsrc_fini(state
, cleanup
);
724 /* Set "status" and "errormsg" and goto failure */
725 TAVOR_TNF_FAIL(DDI_FAILURE
, "QPC table");
726 goto rsrcinitp2_fail
;
728 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL8
;
731 * Initialize the resource pool for the RDB table entries. Notice
732 * that the number of RDBs is configurable. The configured value must
733 * be less that the maximum value (obtained from the QUERY_DEV_LIM
734 * command) or the initialization will fail.
736 num
= ((uint64_t)1 << cfgprof
->cp_log_num_rdb
);
737 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_ra_glob
);
739 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_RDB
];
740 rsrc_pool
->rsrc_type
= TAVOR_RDB
;
741 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
742 rsrc_pool
->rsrc_pool_size
= (TAVOR_RDB_SIZE
* num
);
743 rsrc_pool
->rsrc_shift
= TAVOR_RDB_SIZE_SHIFT
;
744 rsrc_pool
->rsrc_quantum
= TAVOR_RDB_SIZE
;
745 rsrc_pool
->rsrc_align
= (TAVOR_RDB_SIZE
* num
);
746 rsrc_pool
->rsrc_state
= state
;
747 rsrc_pool
->rsrc_start
= NULL
;
748 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_RDB_VMEM
);
749 entry_info
.hwi_num
= num
;
750 entry_info
.hwi_max
= max
;
751 entry_info
.hwi_prealloc
= num_prealloc
;
752 entry_info
.hwi_rsrcpool
= rsrc_pool
;
753 entry_info
.hwi_rsrcname
= rsrc_name
;
754 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
755 if (status
!= DDI_SUCCESS
) {
756 tavor_rsrc_fini(state
, cleanup
);
757 /* Set "status" and "errormsg" and goto failure */
758 TAVOR_TNF_FAIL(DDI_FAILURE
, "RDB table");
759 goto rsrcinitp2_fail
;
761 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL9
;
764 * Initialize the resource pool for the CQC table entries. Notice
765 * that the number of CQs is configurable. The configured value must
766 * be less that the maximum value (obtained from the QUERY_DEV_LIM
767 * command) or the initialization will fail. Note also that a certain
768 * number of CQ contexts must be set aside for Tavor firmware use.
770 num
= ((uint64_t)1 << cfgprof
->cp_log_num_cq
);
771 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_cq
);
772 num_prealloc
= ((uint64_t)1 << state
->ts_devlim
.log_rsvd_cq
);
773 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_CQC
];
774 rsrc_pool
->rsrc_type
= TAVOR_CQC
;
775 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
776 rsrc_pool
->rsrc_pool_size
= (TAVOR_CQC_SIZE
* num
);
777 rsrc_pool
->rsrc_shift
= TAVOR_CQC_SIZE_SHIFT
;
778 rsrc_pool
->rsrc_quantum
= TAVOR_CQC_SIZE
;
779 rsrc_pool
->rsrc_align
= (TAVOR_CQC_SIZE
* num
);
780 rsrc_pool
->rsrc_state
= state
;
781 rsrc_pool
->rsrc_start
= NULL
;
782 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_CQC_VMEM
);
783 entry_info
.hwi_num
= num
;
784 entry_info
.hwi_max
= max
;
785 entry_info
.hwi_prealloc
= num_prealloc
;
786 entry_info
.hwi_rsrcpool
= rsrc_pool
;
787 entry_info
.hwi_rsrcname
= rsrc_name
;
788 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
789 if (status
!= DDI_SUCCESS
) {
790 tavor_rsrc_fini(state
, cleanup
);
791 /* Set "status" and "errormsg" and goto failure */
792 TAVOR_TNF_FAIL(DDI_FAILURE
, "CQC table");
793 goto rsrcinitp2_fail
;
795 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL10
;
798 * Initialize the resource pool for the Extended QPC table entries.
799 * Notice that the number of EQPCs must be the same as the number
800 * of QP contexts. So the initialization is constructed in a
801 * similar way as above (for TAVOR_QPC). One notable difference
802 * here, however, is that by setting the rsrc_quantum field to
803 * zero (indicating a zero-sized object) we indicate that the
804 * object is not allocatable. The EQPC table is, in fact, managed
805 * internally by the hardware and it is, therefore, unnecessary to
806 * initialize an additional vmem_arena for this type of object.
808 num
= ((uint64_t)1 << cfgprof
->cp_log_num_qp
);
809 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_qp
);
811 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_EQPC
];
812 rsrc_pool
->rsrc_type
= TAVOR_EQPC
;
813 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
814 rsrc_pool
->rsrc_pool_size
= (TAVOR_EQPC_SIZE
* num
);
815 rsrc_pool
->rsrc_shift
= 0;
816 rsrc_pool
->rsrc_quantum
= 0;
817 rsrc_pool
->rsrc_align
= TAVOR_EQPC_SIZE
;
818 rsrc_pool
->rsrc_state
= state
;
819 rsrc_pool
->rsrc_start
= NULL
;
820 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_EQPC_VMEM
);
821 entry_info
.hwi_num
= num
;
822 entry_info
.hwi_max
= max
;
823 entry_info
.hwi_prealloc
= num_prealloc
;
824 entry_info
.hwi_rsrcpool
= rsrc_pool
;
825 entry_info
.hwi_rsrcname
= rsrc_name
;
826 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
827 if (status
!= DDI_SUCCESS
) {
828 tavor_rsrc_fini(state
, cleanup
);
829 /* Set "status" and "errormsg" and goto failure */
830 TAVOR_TNF_FAIL(DDI_FAILURE
, "Extended QPC table");
831 goto rsrcinitp2_fail
;
833 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL11
;
836 * Initialize the resource pool for the UD address vector table
837 * entries. Notice that the number of UDAVs is configurable. The
838 * configured value must be less that the maximum value (obtained
839 * from the QUERY_DEV_LIM command) or the initialization will fail.
841 num
= ((uint64_t)1 << cfgprof
->cp_log_num_ah
);
842 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_av
);
844 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_UDAV
];
845 rsrc_pool
->rsrc_type
= TAVOR_UDAV
;
846 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
847 rsrc_pool
->rsrc_pool_size
= (TAVOR_UDAV_SIZE
* num
);
848 rsrc_pool
->rsrc_shift
= TAVOR_UDAV_SIZE_SHIFT
;
849 rsrc_pool
->rsrc_quantum
= TAVOR_UDAV_SIZE
;
850 rsrc_pool
->rsrc_align
= TAVOR_UDAV_SIZE
;
851 rsrc_pool
->rsrc_state
= state
;
852 rsrc_pool
->rsrc_start
= NULL
;
853 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_UDAV_VMEM
);
854 entry_info
.hwi_num
= num
;
855 entry_info
.hwi_max
= max
;
856 entry_info
.hwi_prealloc
= num_prealloc
;
857 entry_info
.hwi_rsrcpool
= rsrc_pool
;
858 entry_info
.hwi_rsrcname
= rsrc_name
;
859 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
860 if (status
!= DDI_SUCCESS
) {
861 tavor_rsrc_fini(state
, cleanup
);
862 /* Set "status" and "errormsg" and goto failure */
863 TAVOR_TNF_FAIL(DDI_FAILURE
, "UDAV table");
864 goto rsrcinitp2_fail
;
866 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL12
;
869 * Initialize the resource pool for the UAR scratch table entries.
870 * Notice that the number of UARSCRs is configurable. The configured
871 * value must be less that the maximum value (obtained from the
872 * QUERY_DEV_LIM command) or the initialization will fail.
873 * Like the EQPCs above, UARSCR objects are not allocatable. The
874 * UARSCR table is also managed internally by the hardware and it
875 * is, therefore, unnecessary to initialize an additional vmem_arena
876 * for this type of object. We indicate this by setting the
877 * rsrc_quantum field to zero (indicating a zero-sized object).
879 uarscr_size
= state
->ts_devlim
.uarscr_entry_sz
;
880 num
= ((uint64_t)1 << cfgprof
->cp_log_num_uar
);
881 max
= ((uint64_t)1 << (state
->ts_devlim
.log_max_uar_sz
+ 20 -
884 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_UAR_SCR
];
885 rsrc_pool
->rsrc_type
= TAVOR_UAR_SCR
;
886 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
887 rsrc_pool
->rsrc_pool_size
= (uarscr_size
* num
);
888 rsrc_pool
->rsrc_shift
= 0;
889 rsrc_pool
->rsrc_quantum
= 0;
890 rsrc_pool
->rsrc_align
= uarscr_size
;
891 rsrc_pool
->rsrc_state
= state
;
892 rsrc_pool
->rsrc_start
= NULL
;
893 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_UARSCR_VMEM
);
894 entry_info
.hwi_num
= num
;
895 entry_info
.hwi_max
= max
;
896 entry_info
.hwi_prealloc
= num_prealloc
;
897 entry_info
.hwi_rsrcpool
= rsrc_pool
;
898 entry_info
.hwi_rsrcname
= rsrc_name
;
899 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
900 if (status
!= DDI_SUCCESS
) {
901 tavor_rsrc_fini(state
, cleanup
);
902 /* Set "status" and "errormsg" and goto failure */
903 TAVOR_TNF_FAIL(DDI_FAILURE
, "UAR scratch table");
904 goto rsrcinitp2_fail
;
906 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL13
;
909 * Initialize the resource pool for the SRQC table entries. Notice
910 * that the number of SRQs is configurable. The configured value must
911 * be less that the maximum value (obtained from the QUERY_DEV_LIM
912 * command) or the initialization will fail. Note also that a certain
913 * number of SRQ contexts must be set aside for Tavor firmware use.
915 * Note: We only allocate these resources if SRQ is enabled in the
916 * config profile; see below.
918 num
= ((uint64_t)1 << cfgprof
->cp_log_num_srq
);
919 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_srq
);
920 num_prealloc
= ((uint64_t)1 << state
->ts_devlim
.log_rsvd_srq
);
922 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_SRQC
];
923 rsrc_pool
->rsrc_type
= TAVOR_SRQC
;
924 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
925 rsrc_pool
->rsrc_pool_size
= (TAVOR_SRQC_SIZE
* num
);
926 rsrc_pool
->rsrc_shift
= TAVOR_SRQC_SIZE_SHIFT
;
927 rsrc_pool
->rsrc_quantum
= TAVOR_SRQC_SIZE
;
928 rsrc_pool
->rsrc_align
= (TAVOR_SRQC_SIZE
* num
);
929 rsrc_pool
->rsrc_state
= state
;
930 rsrc_pool
->rsrc_start
= NULL
;
931 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_SRQC_VMEM
);
932 entry_info
.hwi_num
= num
;
933 entry_info
.hwi_max
= max
;
934 entry_info
.hwi_prealloc
= num_prealloc
;
935 entry_info
.hwi_rsrcpool
= rsrc_pool
;
936 entry_info
.hwi_rsrcname
= rsrc_name
;
939 * SRQ support is configurable. Only if SRQ is enabled (the default)
940 * do we actually try to configure these resources. Otherwise, we
941 * simply set the cleanup level and continue on to the next resource
943 if (state
->ts_cfg_profile
->cp_srq_enable
!= 0) {
944 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
945 if (status
!= DDI_SUCCESS
) {
946 tavor_rsrc_fini(state
, cleanup
);
947 /* Set "status" and "errormsg" and goto failure */
948 TAVOR_TNF_FAIL(DDI_FAILURE
, "SRQC table");
949 goto rsrcinitp2_fail
;
952 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL14
;
955 * Initialize the resource pool for "In" mailboxes. Notice that
956 * the number of "In" mailboxes, their size, and their location
957 * (DDR or system memory) is configurable. By default, however,
958 * all "In" mailboxes are located in system memory only (because
959 * they are primarily written to and rarely read from)
961 num
= ((uint64_t)1 << cfgprof
->cp_log_num_inmbox
);
962 size
= ((uint64_t)1 << cfgprof
->cp_log_inmbox_size
);
963 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_IN_MBOX
];
964 rsrc_pool
->rsrc_type
= TAVOR_IN_MBOX
;
965 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
966 rsrc_pool
->rsrc_pool_size
= (size
* num
);
967 rsrc_pool
->rsrc_shift
= cfgprof
->cp_log_inmbox_size
;
968 rsrc_pool
->rsrc_quantum
= size
;
969 rsrc_pool
->rsrc_align
= TAVOR_MBOX_ALIGN
;
970 rsrc_pool
->rsrc_state
= state
;
971 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_INMBOX_VMEM
);
972 mbox_info
.mbi_num
= num
;
973 mbox_info
.mbi_size
= size
;
974 mbox_info
.mbi_rsrcpool
= rsrc_pool
;
975 mbox_info
.mbi_rsrcname
= rsrc_name
;
976 status
= tavor_rsrc_mbox_init(state
, &mbox_info
);
977 if (status
!= DDI_SUCCESS
) {
978 tavor_rsrc_fini(state
, cleanup
);
979 /* Set "status" and "errormsg" and goto failure */
980 TAVOR_TNF_FAIL(DDI_FAILURE
, "in mailboxes");
981 goto rsrcinitp2_fail
;
983 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL15
;
986 * Initialize the Tavor "In" mailbox list. This step actually uses
987 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
988 * "In" mailboxes, bind them for DMA access, and arrange them into
989 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
992 status
= tavor_inmbox_list_init(state
);
993 if (status
!= DDI_SUCCESS
) {
994 tavor_rsrc_fini(state
, cleanup
);
995 /* Set "status" and "errormsg" and goto failure */
996 TAVOR_TNF_FAIL(DDI_FAILURE
, "in mailbox list");
997 goto rsrcinitp2_fail
;
999 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL16
;
1002 * Initialize the resource pool for interrupt "In" mailboxes. Notice
1003 * that the number of interrupt "In" mailboxes, their size, and their
1004 * location (DDR or system memory) is configurable. By default,
1005 * however, all interrupt "In" mailboxes are located in system memory
1006 * only (because they are primarily written to and rarely read from)
1008 num
= ((uint64_t)1 << cfgprof
->cp_log_num_intr_inmbox
);
1009 size
= ((uint64_t)1 << cfgprof
->cp_log_inmbox_size
);
1010 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_INTR_IN_MBOX
];
1011 rsrc_pool
->rsrc_type
= TAVOR_INTR_IN_MBOX
;
1012 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
1013 rsrc_pool
->rsrc_pool_size
= (size
* num
);
1014 rsrc_pool
->rsrc_shift
= cfgprof
->cp_log_inmbox_size
;
1015 rsrc_pool
->rsrc_quantum
= size
;
1016 rsrc_pool
->rsrc_align
= TAVOR_MBOX_ALIGN
;
1017 rsrc_pool
->rsrc_state
= state
;
1018 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_INTR_INMBOX_VMEM
);
1019 mbox_info
.mbi_num
= num
;
1020 mbox_info
.mbi_size
= size
;
1021 mbox_info
.mbi_rsrcpool
= rsrc_pool
;
1022 mbox_info
.mbi_rsrcname
= rsrc_name
;
1023 status
= tavor_rsrc_mbox_init(state
, &mbox_info
);
1024 if (status
!= DDI_SUCCESS
) {
1025 tavor_rsrc_fini(state
, cleanup
);
1026 /* Set "status" and "errormsg" and goto failure */
1027 TAVOR_TNF_FAIL(DDI_FAILURE
, "in intr mailboxes");
1028 goto rsrcinitp2_fail
;
1030 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL17
;
1033 * Initialize the Tavor interrupt "In" mailbox list. This step
1034 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
1035 * preallocate the interrupt "In" mailboxes, bind them for DMA access,
1036 * and arrange them into an easily accessed fast-allocation mechanism
1037 * (see tavor_cmd.c for more details)
1039 status
= tavor_intr_inmbox_list_init(state
);
1040 if (status
!= DDI_SUCCESS
) {
1041 tavor_rsrc_fini(state
, cleanup
);
1042 /* Set "status" and "errormsg" and goto failure */
1043 TAVOR_TNF_FAIL(DDI_FAILURE
, "in intr mailbox list");
1044 goto rsrcinitp2_fail
;
1046 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL18
;
1049 * Initialize the Tavor command handling interfaces. This step
1050 * sets up the outstanding command tracking mechanism for easy access
1051 * and fast allocation (see tavor_cmd.c for more details).
1053 status
= tavor_outstanding_cmdlist_init(state
);
1054 if (status
!= DDI_SUCCESS
) {
1055 tavor_rsrc_fini(state
, cleanup
);
1056 /* Set "status" and "errormsg" and goto failure */
1057 TAVOR_TNF_FAIL(DDI_FAILURE
, "outstanding cmd list");
1058 goto rsrcinitp2_fail
;
1060 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL19
;
1063 * Calculate (and validate) the size of Multicast Group (MCG) entries
1065 status
= tavor_rsrc_mcg_entry_get_size(state
, &mcg_size_shift
);
1066 if (status
!= DDI_SUCCESS
) {
1067 tavor_rsrc_fini(state
, cleanup
);
1068 /* Set "status" and "errormsg" and goto failure */
1069 TAVOR_TNF_FAIL(DDI_FAILURE
, "failed get MCG size");
1070 goto rsrcinitp2_fail
;
1072 mcg_size
= TAVOR_MCGMEM_SZ(state
);
1075 * Initialize the resource pool for the MCG table entries. Notice
1076 * that the number of MCGs is configurable. The configured value must
1077 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1078 * command) or the initialization will fail. Note also that a certain
1079 * number of MCGs must be set aside for Tavor firmware use (they
1080 * correspond to the number of MCGs used by the internal hash
1083 num
= ((uint64_t)1 << cfgprof
->cp_log_num_mcg
);
1084 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_mcg
);
1085 num_prealloc
= ((uint64_t)1 << cfgprof
->cp_log_num_mcg_hash
);
1086 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_MCG
];
1087 rsrc_pool
->rsrc_type
= TAVOR_MCG
;
1088 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
1089 rsrc_pool
->rsrc_pool_size
= (mcg_size
* num
);
1090 rsrc_pool
->rsrc_shift
= mcg_size_shift
;
1091 rsrc_pool
->rsrc_quantum
= mcg_size
;
1092 rsrc_pool
->rsrc_align
= mcg_size
;
1093 rsrc_pool
->rsrc_state
= state
;
1094 rsrc_pool
->rsrc_start
= NULL
;
1095 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_MCG_VMEM
);
1096 entry_info
.hwi_num
= num
;
1097 entry_info
.hwi_max
= max
;
1098 entry_info
.hwi_prealloc
= num_prealloc
;
1099 entry_info
.hwi_rsrcpool
= rsrc_pool
;
1100 entry_info
.hwi_rsrcname
= rsrc_name
;
1101 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
1102 if (status
!= DDI_SUCCESS
) {
1103 tavor_rsrc_fini(state
, cleanup
);
1104 /* Set "status" and "errormsg" and goto failure */
1105 TAVOR_TNF_FAIL(DDI_FAILURE
, "MCG table");
1106 goto rsrcinitp2_fail
;
1108 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL20
;
1111 * Initialize the resource pool for the EQC table entries. Notice
1112 * that the number of EQs is hardcoded. The hardcoded value should
1113 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1114 * command) or the initialization will fail.
1117 max
= ((uint64_t)1 << state
->ts_devlim
.log_max_eq
);
1119 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_EQC
];
1120 rsrc_pool
->rsrc_type
= TAVOR_EQC
;
1121 rsrc_pool
->rsrc_loc
= TAVOR_IN_DDR
;
1122 rsrc_pool
->rsrc_pool_size
= (TAVOR_EQC_SIZE
* num
);
1123 rsrc_pool
->rsrc_shift
= TAVOR_EQC_SIZE_SHIFT
;
1124 rsrc_pool
->rsrc_quantum
= TAVOR_EQC_SIZE
;
1125 rsrc_pool
->rsrc_align
= (TAVOR_EQC_SIZE
* num
);
1126 rsrc_pool
->rsrc_state
= state
;
1127 rsrc_pool
->rsrc_start
= NULL
;
1128 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_DDR_EQC_VMEM
);
1129 entry_info
.hwi_num
= num
;
1130 entry_info
.hwi_max
= max
;
1131 entry_info
.hwi_prealloc
= num_prealloc
;
1132 entry_info
.hwi_rsrcpool
= rsrc_pool
;
1133 entry_info
.hwi_rsrcname
= rsrc_name
;
1134 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
1135 if (status
!= DDI_SUCCESS
) {
1136 tavor_rsrc_fini(state
, cleanup
);
1137 /* Set "status" and "errormsg" and goto failure */
1138 TAVOR_TNF_FAIL(DDI_FAILURE
, "EQC table");
1139 goto rsrcinitp2_fail
;
1141 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL21
;
1144 * Initialize the resource pools for all objects that exist in
1145 * system memory. This includes PD handles, MR handle, EQ handles,
1146 * QP handles, etc. These objects are almost entirely managed using
1147 * kmem_cache routines. (See comment above for more detail)
1151 * Initialize the resource pool for the PD handles. Notice
1152 * that the number of PDHDLs is configurable. The configured value
1153 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1154 * command) or the initialization will fail. Note also that the PD
1155 * handle has constructor and destructor methods associated with it.
1157 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_PDHDL
];
1158 rsrc_pool
->rsrc_type
= TAVOR_PDHDL
;
1159 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1160 rsrc_pool
->rsrc_quantum
= sizeof (struct tavor_sw_pd_s
);
1161 rsrc_pool
->rsrc_state
= state
;
1162 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_PDHDL_CACHE
);
1163 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_pd
);
1164 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_pd
);
1165 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1166 hdl_info
.swi_constructor
= tavor_rsrc_pdhdl_constructor
;
1167 hdl_info
.swi_destructor
= tavor_rsrc_pdhdl_destructor
;
1168 hdl_info
.swi_rsrcname
= rsrc_name
;
1169 hdl_info
.swi_flags
= TAVOR_SWHDL_KMEMCACHE_INIT
;
1170 status
= tavor_rsrc_pd_handles_init(state
, &hdl_info
);
1171 if (status
!= DDI_SUCCESS
) {
1172 tavor_rsrc_fini(state
, cleanup
);
1173 /* Set "status" and "errormsg" and goto failure */
1174 TAVOR_TNF_FAIL(DDI_FAILURE
, "PD handle");
1175 goto rsrcinitp2_fail
;
1177 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL22
;
1180 * Initialize the resource pool for the MR handles. Notice
1181 * that the number of MRHDLs is configurable. The configured value
1182 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1183 * command) or the initialization will fail.
1185 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_MRHDL
];
1186 rsrc_pool
->rsrc_type
= TAVOR_MRHDL
;
1187 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1188 rsrc_pool
->rsrc_quantum
= sizeof (struct tavor_sw_mr_s
);
1189 rsrc_pool
->rsrc_state
= state
;
1190 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_MRHDL_CACHE
);
1191 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_mpt
);
1192 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_mpt
);
1193 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1194 hdl_info
.swi_constructor
= tavor_rsrc_mrhdl_constructor
;
1195 hdl_info
.swi_destructor
= tavor_rsrc_mrhdl_destructor
;
1196 hdl_info
.swi_rsrcname
= rsrc_name
;
1197 hdl_info
.swi_flags
= TAVOR_SWHDL_KMEMCACHE_INIT
;
1198 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1199 if (status
!= DDI_SUCCESS
) {
1200 tavor_rsrc_fini(state
, cleanup
);
1201 /* Set "status" and "errormsg" and goto failure */
1202 TAVOR_TNF_FAIL(DDI_FAILURE
, "MR handle");
1203 goto rsrcinitp2_fail
;
1205 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL23
;
1208 * Initialize the resource pool for the EQ handles. Notice
1209 * that the number of EQHDLs is hardcoded. The hardcoded value
1210 * should be less that the maximum value (obtained from the
1211 * QUERY_DEV_LIM command) or the initialization will fail.
1213 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_EQHDL
];
1214 rsrc_pool
->rsrc_type
= TAVOR_EQHDL
;
1215 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1216 rsrc_pool
->rsrc_quantum
= sizeof (struct tavor_sw_eq_s
);
1217 rsrc_pool
->rsrc_state
= state
;
1218 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_EQHDL_CACHE
);
1219 hdl_info
.swi_num
= TAVOR_NUM_EQ
;
1220 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_eq
);
1221 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1222 hdl_info
.swi_constructor
= NULL
;
1223 hdl_info
.swi_destructor
= NULL
;
1224 hdl_info
.swi_rsrcname
= rsrc_name
;
1225 hdl_info
.swi_flags
= TAVOR_SWHDL_KMEMCACHE_INIT
;
1226 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1227 if (status
!= DDI_SUCCESS
) {
1228 tavor_rsrc_fini(state
, cleanup
);
1229 /* Set "status" and "errormsg" and goto failure */
1230 TAVOR_TNF_FAIL(DDI_FAILURE
, "EQ handle");
1231 goto rsrcinitp2_fail
;
1233 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL24
;
1236 * Initialize the resource pool for the CQ handles. Notice
1237 * that the number of CQHDLs is configurable. The configured value
1238 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1239 * command) or the initialization will fail. Note also that the CQ
1240 * handle has constructor and destructor methods associated with it.
1242 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_CQHDL
];
1243 rsrc_pool
->rsrc_type
= TAVOR_CQHDL
;
1244 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1245 rsrc_pool
->rsrc_quantum
= sizeof (struct tavor_sw_cq_s
);
1246 rsrc_pool
->rsrc_state
= state
;
1247 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_CQHDL_CACHE
);
1248 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_cq
);
1249 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_cq
);
1250 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1251 hdl_info
.swi_constructor
= tavor_rsrc_cqhdl_constructor
;
1252 hdl_info
.swi_destructor
= tavor_rsrc_cqhdl_destructor
;
1253 hdl_info
.swi_rsrcname
= rsrc_name
;
1254 hdl_info
.swi_flags
= (TAVOR_SWHDL_KMEMCACHE_INIT
|
1255 TAVOR_SWHDL_TABLE_INIT
);
1256 hdl_info
.swi_prealloc_sz
= sizeof (tavor_cqhdl_t
);
1257 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1258 if (status
!= DDI_SUCCESS
) {
1259 tavor_rsrc_fini(state
, cleanup
);
1260 /* Set "status" and "errormsg" and goto failure */
1261 TAVOR_TNF_FAIL(DDI_FAILURE
, "CQ handle");
1262 goto rsrcinitp2_fail
;
1266 * Save away the pointer to the central list of CQ handle pointers
1267 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
1268 * lookup during EQ event processing. The table is a list of
1269 * tavor_cqhdl_t allocated by the above routine because of the
1270 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t
1271 * as the number of CQs.
1273 state
->ts_cqhdl
= hdl_info
.swi_table_ptr
;
1274 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL25
;
1277 * Initialize the resource pool for the SRQ handles. Notice
1278 * that the number of SRQHDLs is configurable. The configured value
1279 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1280 * command) or the initialization will fail. Note also that the SRQ
1281 * handle has constructor and destructor methods associated with it.
1283 * Note: We only allocate these resources if SRQ is enabled in the
1284 * config profile; see below.
1286 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_SRQHDL
];
1287 rsrc_pool
->rsrc_type
= TAVOR_SRQHDL
;
1288 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1289 rsrc_pool
->rsrc_quantum
= sizeof (struct tavor_sw_srq_s
);
1290 rsrc_pool
->rsrc_state
= state
;
1291 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_SRQHDL_CACHE
);
1292 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_srq
);
1293 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_srq
);
1294 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1295 hdl_info
.swi_constructor
= tavor_rsrc_srqhdl_constructor
;
1296 hdl_info
.swi_destructor
= tavor_rsrc_srqhdl_destructor
;
1297 hdl_info
.swi_rsrcname
= rsrc_name
;
1298 hdl_info
.swi_flags
= (TAVOR_SWHDL_KMEMCACHE_INIT
|
1299 TAVOR_SWHDL_TABLE_INIT
);
1300 hdl_info
.swi_prealloc_sz
= sizeof (tavor_srqhdl_t
);
1303 * SRQ support is configurable. Only if SRQ is enabled (the default)
1304 * do we actually try to configure these resources. Otherwise, we
1305 * simply set the cleanup level and continue on to the next resource
1307 if (state
->ts_cfg_profile
->cp_srq_enable
!= 0) {
1308 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1309 if (status
!= DDI_SUCCESS
) {
1310 tavor_rsrc_fini(state
, cleanup
);
1311 /* Set "status" and "errormsg" and goto failure */
1312 TAVOR_TNF_FAIL(DDI_FAILURE
, "SRQ handle");
1313 goto rsrcinitp2_fail
;
1317 * Save away the pointer to the central list of SRQ handle
1318 * pointers This this is used as a mechanism to enable fast
1319 * SRQnumber-to-SRQhandle lookup. The table is a list of
1320 * tavor_srqhdl_t allocated by the above routine because of the
1321 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many
1322 * tavor_srqhdl_t as the number of SRQs.
1324 state
->ts_srqhdl
= hdl_info
.swi_table_ptr
;
1326 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL26
;
1329 * Initialize the resource pool for the address handles. Notice
1330 * that the number of AHHDLs is configurable. The configured value
1331 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1332 * command) or the initialization will fail.
1334 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_AHHDL
];
1335 rsrc_pool
->rsrc_type
= TAVOR_AHHDL
;
1336 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1337 rsrc_pool
->rsrc_quantum
= sizeof (struct tavor_sw_ah_s
);
1338 rsrc_pool
->rsrc_state
= state
;
1339 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_AHHDL_CACHE
);
1340 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_ah
);
1341 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_av
);
1342 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1343 hdl_info
.swi_constructor
= tavor_rsrc_ahhdl_constructor
;
1344 hdl_info
.swi_destructor
= tavor_rsrc_ahhdl_destructor
;
1345 hdl_info
.swi_rsrcname
= rsrc_name
;
1346 hdl_info
.swi_flags
= TAVOR_SWHDL_KMEMCACHE_INIT
;
1347 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1348 if (status
!= DDI_SUCCESS
) {
1349 tavor_rsrc_fini(state
, cleanup
);
1350 /* Set "status" and "errormsg" and goto failure */
1351 TAVOR_TNF_FAIL(DDI_FAILURE
, "AH handle");
1352 goto rsrcinitp2_fail
;
1354 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL27
;
1357 * Initialize the resource pool for the QP handles. Notice
1358 * that the number of QPHDLs is configurable. The configured value
1359 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1360 * command) or the initialization will fail. Note also that the QP
1361 * handle has constructor and destructor methods associated with it.
1363 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_QPHDL
];
1364 rsrc_pool
->rsrc_type
= TAVOR_QPHDL
;
1365 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1366 rsrc_pool
->rsrc_quantum
= sizeof (struct tavor_sw_qp_s
);
1367 rsrc_pool
->rsrc_state
= state
;
1368 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_QPHDL_CACHE
);
1369 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_qp
);
1370 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_qp
);
1371 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1372 hdl_info
.swi_constructor
= tavor_rsrc_qphdl_constructor
;
1373 hdl_info
.swi_destructor
= tavor_rsrc_qphdl_destructor
;
1374 hdl_info
.swi_rsrcname
= rsrc_name
;
1375 hdl_info
.swi_flags
= (TAVOR_SWHDL_KMEMCACHE_INIT
|
1376 TAVOR_SWHDL_TABLE_INIT
);
1377 hdl_info
.swi_prealloc_sz
= sizeof (tavor_qphdl_t
);
1378 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1379 if (status
!= DDI_SUCCESS
) {
1380 tavor_rsrc_fini(state
, cleanup
);
1381 /* Set "status" and "errormsg" and goto failure */
1382 TAVOR_TNF_FAIL(DDI_FAILURE
, "QP handle");
1383 goto rsrcinitp2_fail
;
1387 * Save away the pointer to the central list of QP handle pointers
1388 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
1389 * lookup during CQ event processing. The table is a list of
1390 * tavor_qphdl_t allocated by the above routine because of the
1391 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t
1392 * as the number of QPs.
1394 state
->ts_qphdl
= hdl_info
.swi_table_ptr
;
1395 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL28
;
1398 * Initialize the resource pool for the reference count handles.
1399 * Notice that the number of REFCNTs is configurable, but it's value
1400 * is set to the number of MPTs. Since REFCNTs are used to support
1401 * shared memory regions, it is possible that we might require as
1402 * one REFCNT for every MPT.
1404 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_REFCNT
];
1405 rsrc_pool
->rsrc_type
= TAVOR_REFCNT
;
1406 rsrc_pool
->rsrc_loc
= TAVOR_IN_SYSMEM
;
1407 rsrc_pool
->rsrc_quantum
= sizeof (tavor_sw_refcnt_t
);
1408 rsrc_pool
->rsrc_state
= state
;
1409 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_REFCNT_CACHE
);
1410 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_mpt
);
1411 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_mpt
);
1412 hdl_info
.swi_rsrcpool
= rsrc_pool
;
1413 hdl_info
.swi_constructor
= tavor_rsrc_refcnt_constructor
;
1414 hdl_info
.swi_destructor
= tavor_rsrc_refcnt_destructor
;
1415 hdl_info
.swi_rsrcname
= rsrc_name
;
1416 hdl_info
.swi_flags
= TAVOR_SWHDL_KMEMCACHE_INIT
;
1417 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1418 if (status
!= DDI_SUCCESS
) {
1419 tavor_rsrc_fini(state
, cleanup
);
1420 /* Set "status" and "errormsg" and goto failure */
1421 TAVOR_TNF_FAIL(DDI_FAILURE
, "reference count handle");
1422 goto rsrcinitp2_fail
;
1424 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL29
;
1427 * Initialize the resource pool for the MCG handles. Notice that for
1428 * these MCG handles, we are allocating a table of structures (used to
1429 * keep track of the MCG entries that are being written to hardware
1430 * and to speed up multicast attach/detach operations).
1432 hdl_info
.swi_num
= ((uint64_t)1 << cfgprof
->cp_log_num_mcg
);
1433 hdl_info
.swi_max
= ((uint64_t)1 << state
->ts_devlim
.log_max_mcg
);
1434 hdl_info
.swi_flags
= TAVOR_SWHDL_TABLE_INIT
;
1435 hdl_info
.swi_prealloc_sz
= sizeof (struct tavor_sw_mcg_list_s
);
1436 status
= tavor_rsrc_sw_handles_init(state
, &hdl_info
);
1437 if (status
!= DDI_SUCCESS
) {
1438 tavor_rsrc_fini(state
, cleanup
);
1439 /* Set "status" and "errormsg" and goto failure */
1440 TAVOR_TNF_FAIL(DDI_FAILURE
, "MCG handle");
1441 goto rsrcinitp2_fail
;
1443 state
->ts_mcghdl
= hdl_info
.swi_table_ptr
;
1444 cleanup
= TAVOR_RSRC_CLEANUP_LEVEL30
;
1447 * Initialize the resource pools for all objects that exist in
1448 * UAR memory. The only objects that are allocated from UAR memory
1449 * are the UAR pages which are used for holding Tavor hardware's
1450 * doorbell registers.
1454 * Initialize the resource pool for the UAR pages. Notice
1455 * that the number of UARPGs is configurable. The configured value
1456 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1457 * command) or the initialization will fail. Note also that by
1458 * specifying the rsrc_start parameter in advance, we direct the
1459 * initialization routine not to attempt to allocated space from the
1460 * Tavor DDR vmem_arena.
1462 num
= ((uint64_t)1 << cfgprof
->cp_log_num_uar
);
1463 max
= ((uint64_t)1 << (state
->ts_devlim
.log_max_uar_sz
+ 20 -
1466 rsrc_pool
= &state
->ts_rsrc_hdl
[TAVOR_UARPG
];
1467 rsrc_pool
->rsrc_type
= TAVOR_UARPG
;
1468 rsrc_pool
->rsrc_loc
= TAVOR_IN_UAR
;
1469 rsrc_pool
->rsrc_pool_size
= (num
<< PAGESHIFT
);
1470 rsrc_pool
->rsrc_shift
= PAGESHIFT
;
1471 rsrc_pool
->rsrc_quantum
= PAGESIZE
;
1472 rsrc_pool
->rsrc_align
= PAGESIZE
;
1473 rsrc_pool
->rsrc_state
= state
;
1474 rsrc_pool
->rsrc_start
= (void *)state
->ts_reg_uar_baseaddr
;
1475 TAVOR_RSRC_NAME(rsrc_name
, TAVOR_UAR_VMEM
);
1476 entry_info
.hwi_num
= num
;
1477 entry_info
.hwi_max
= max
;
1478 entry_info
.hwi_prealloc
= num_prealloc
;
1479 entry_info
.hwi_rsrcpool
= rsrc_pool
;
1480 entry_info
.hwi_rsrcname
= rsrc_name
;
1481 status
= tavor_rsrc_hw_entries_init(state
, &entry_info
);
1482 if (status
!= DDI_SUCCESS
) {
1483 tavor_rsrc_fini(state
, cleanup
);
1484 /* Set "status" and "errormsg" and goto failure */
1485 TAVOR_TNF_FAIL(DDI_FAILURE
, "UAR page table");
1486 goto rsrcinitp2_fail
;
1488 cleanup
= TAVOR_RSRC_CLEANUP_ALL
;
1490 kmem_free(rsrc_name
, TAVOR_RSRC_NAME_MAXLEN
);
1491 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2
);
1492 return (DDI_SUCCESS
);
1495 kmem_free(rsrc_name
, TAVOR_RSRC_NAME_MAXLEN
);
1496 TNF_PROBE_1(tavor_rsrc_init_phase2_fail
, TAVOR_TNF_ERROR
, "",
1497 tnf_string
, msg
, errormsg
);
1498 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2
);
1505 * Context: Only called from attach() and/or detach() path contexts
1508 tavor_rsrc_fini(tavor_state_t
*state
, tavor_rsrc_cleanup_level_t clean
)
1510 tavor_rsrc_sw_hdl_info_t hdl_info
;
1511 tavor_rsrc_hw_entry_info_t entry_info
;
1512 tavor_rsrc_mbox_info_t mbox_info
;
1513 tavor_cfg_profile_t
*cfgprof
;
1515 TAVOR_TNF_ENTER(tavor_rsrc_fini
);
1517 ASSERT(state
!= NULL
);
1519 cfgprof
= state
->ts_cfg_profile
;
1523 * If we add more resources that need to be cleaned up here, we should
1524 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
1525 * corresponds to the last resource allocated).
1527 case TAVOR_RSRC_CLEANUP_ALL
:
1528 /* Cleanup the UAR page resource pool */
1529 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_UARPG
];
1530 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1533 case TAVOR_RSRC_CLEANUP_LEVEL30
:
1534 /* Cleanup the central MCG handle pointers list */
1535 hdl_info
.swi_rsrcpool
= NULL
;
1536 hdl_info
.swi_table_ptr
= state
->ts_mcghdl
;
1538 ((uint64_t)1 << cfgprof
->cp_log_num_mcg
);
1539 hdl_info
.swi_prealloc_sz
= sizeof (struct tavor_sw_mcg_list_s
);
1540 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1543 case TAVOR_RSRC_CLEANUP_LEVEL29
:
1544 /* Cleanup the reference count resource pool */
1545 hdl_info
.swi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_REFCNT
];
1546 hdl_info
.swi_table_ptr
= NULL
;
1547 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1550 case TAVOR_RSRC_CLEANUP_LEVEL28
:
1551 /* Cleanup the QP handle resource pool */
1552 hdl_info
.swi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_QPHDL
];
1553 hdl_info
.swi_table_ptr
= state
->ts_qphdl
;
1555 ((uint64_t)1 << cfgprof
->cp_log_num_qp
);
1556 hdl_info
.swi_prealloc_sz
= sizeof (tavor_qphdl_t
);
1557 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1560 case TAVOR_RSRC_CLEANUP_LEVEL27
:
1561 /* Cleanup the address handle resource pool */
1562 hdl_info
.swi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_AHHDL
];
1563 hdl_info
.swi_table_ptr
= NULL
;
1564 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1567 case TAVOR_RSRC_CLEANUP_LEVEL26
:
1569 * Cleanup the SRQ handle resource pool.
1571 * Note: We only clean up if SRQ is enabled. Otherwise we
1572 * simply fallthrough to the next resource cleanup.
1574 if (state
->ts_cfg_profile
->cp_srq_enable
!= 0) {
1575 hdl_info
.swi_rsrcpool
=
1576 &state
->ts_rsrc_hdl
[TAVOR_SRQHDL
];
1577 hdl_info
.swi_table_ptr
= state
->ts_srqhdl
;
1579 ((uint64_t)1 << cfgprof
->cp_log_num_srq
);
1580 hdl_info
.swi_prealloc_sz
= sizeof (tavor_srqhdl_t
);
1581 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1585 case TAVOR_RSRC_CLEANUP_LEVEL25
:
1586 /* Cleanup the CQ handle resource pool */
1587 hdl_info
.swi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_CQHDL
];
1588 hdl_info
.swi_table_ptr
= state
->ts_cqhdl
;
1590 ((uint64_t)1 << cfgprof
->cp_log_num_cq
);
1591 hdl_info
.swi_prealloc_sz
= sizeof (tavor_cqhdl_t
);
1592 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1595 case TAVOR_RSRC_CLEANUP_LEVEL24
:
1596 /* Cleanup the EQ handle resource pool */
1597 hdl_info
.swi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_EQHDL
];
1598 hdl_info
.swi_table_ptr
= NULL
;
1599 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1602 case TAVOR_RSRC_CLEANUP_LEVEL23
:
1603 /* Cleanup the MR handle resource pool */
1604 hdl_info
.swi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_MRHDL
];
1605 hdl_info
.swi_table_ptr
= NULL
;
1606 tavor_rsrc_sw_handles_fini(state
, &hdl_info
);
1609 case TAVOR_RSRC_CLEANUP_LEVEL22
:
1610 /* Cleanup the PD handle resource pool */
1611 hdl_info
.swi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_PDHDL
];
1612 hdl_info
.swi_table_ptr
= NULL
;
1613 tavor_rsrc_pd_handles_fini(state
, &hdl_info
);
1616 case TAVOR_RSRC_CLEANUP_LEVEL21
:
1617 /* Cleanup the EQC table resource pool */
1618 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_EQC
];
1619 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1622 case TAVOR_RSRC_CLEANUP_LEVEL20
:
1623 /* Cleanup the MCG table resource pool */
1624 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_MCG
];
1625 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1628 case TAVOR_RSRC_CLEANUP_LEVEL19
:
1629 /* Cleanup the outstanding command list */
1630 tavor_outstanding_cmdlist_fini(state
);
1633 case TAVOR_RSRC_CLEANUP_LEVEL18
:
1634 /* Cleanup the "In" mailbox list */
1635 tavor_intr_inmbox_list_fini(state
);
1638 case TAVOR_RSRC_CLEANUP_LEVEL17
:
1639 /* Cleanup the interrupt "In" mailbox resource pool */
1640 mbox_info
.mbi_rsrcpool
= &state
->ts_rsrc_hdl
[
1641 TAVOR_INTR_IN_MBOX
];
1642 tavor_rsrc_mbox_fini(state
, &mbox_info
);
1645 case TAVOR_RSRC_CLEANUP_LEVEL16
:
1646 /* Cleanup the "In" mailbox list */
1647 tavor_inmbox_list_fini(state
);
1650 case TAVOR_RSRC_CLEANUP_LEVEL15
:
1651 /* Cleanup the "In" mailbox resource pool */
1652 mbox_info
.mbi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_IN_MBOX
];
1653 tavor_rsrc_mbox_fini(state
, &mbox_info
);
1656 case TAVOR_RSRC_CLEANUP_LEVEL14
:
1658 * Cleanup the SRQC table resource pool.
1660 * Note: We only clean up if SRQ is enabled. Otherwise we
1661 * simply fallthrough to the next resource cleanup.
1663 if (state
->ts_cfg_profile
->cp_srq_enable
!= 0) {
1664 entry_info
.hwi_rsrcpool
=
1665 &state
->ts_rsrc_hdl
[TAVOR_SRQC
];
1666 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1670 case TAVOR_RSRC_CLEANUP_LEVEL13
:
1671 /* Cleanup the UAR scratch table resource pool */
1672 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_UAR_SCR
];
1673 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1676 case TAVOR_RSRC_CLEANUP_LEVEL12
:
1677 /* Cleanup the UDAV table resource pool */
1678 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_UDAV
];
1679 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1682 case TAVOR_RSRC_CLEANUP_LEVEL11
:
1683 /* Cleanup the EQPC table resource pool */
1684 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_EQPC
];
1685 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1688 case TAVOR_RSRC_CLEANUP_LEVEL10
:
1689 /* Cleanup the CQC table resource pool */
1690 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_CQC
];
1691 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1694 case TAVOR_RSRC_CLEANUP_LEVEL9
:
1695 /* Cleanup the RDB table resource pool */
1696 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_RDB
];
1697 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1700 case TAVOR_RSRC_CLEANUP_LEVEL8
:
1701 /* Cleanup the QPC table resource pool */
1702 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_QPC
];
1703 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1706 case TAVOR_RSRC_CLEANUP_LEVEL7
:
1707 /* Cleanup the MTT table resource pool */
1708 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_MTT
];
1709 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1712 case TAVOR_RSRC_CLEANUP_LEVEL6
:
1713 /* Cleanup the MPT table resource pool */
1714 entry_info
.hwi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_MPT
];
1715 tavor_rsrc_hw_entries_fini(state
, &entry_info
);
1718 case TAVOR_RSRC_CLEANUP_LEVEL5
:
1719 /* Destroy the vmem arena for DDR memory */
1720 vmem_destroy(state
->ts_ddrvmem
);
1724 * The cleanup below comes from the "Phase 1" initialization step.
1725 * (see tavor_rsrc_init_phase1() above)
1727 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE
:
1728 /* Cleanup the interrupt "Out" mailbox list */
1729 tavor_intr_outmbox_list_fini(state
);
1732 case TAVOR_RSRC_CLEANUP_LEVEL3
:
1733 /* Cleanup the "Out" mailbox resource pool */
1734 mbox_info
.mbi_rsrcpool
= &state
->ts_rsrc_hdl
[
1735 TAVOR_INTR_OUT_MBOX
];
1736 tavor_rsrc_mbox_fini(state
, &mbox_info
);
1739 case TAVOR_RSRC_CLEANUP_LEVEL2
:
1740 /* Cleanup the "Out" mailbox list */
1741 tavor_outmbox_list_fini(state
);
1744 case TAVOR_RSRC_CLEANUP_LEVEL1
:
1745 /* Cleanup the "Out" mailbox resource pool */
1746 mbox_info
.mbi_rsrcpool
= &state
->ts_rsrc_hdl
[TAVOR_OUT_MBOX
];
1747 tavor_rsrc_mbox_fini(state
, &mbox_info
);
1750 case TAVOR_RSRC_CLEANUP_LEVEL0
:
1751 /* Free the array of tavor_rsrc_pool_info_t's */
1752 kmem_free(state
->ts_rsrc_hdl
, TAVOR_NUM_RESOURCES
*
1753 sizeof (tavor_rsrc_pool_info_t
));
1754 kmem_cache_destroy(state
->ts_rsrc_cache
);
1758 TAVOR_WARNING(state
, "unexpected resource cleanup level");
1759 TNF_PROBE_0(tavor_rsrc_fini_default_fail
, TAVOR_TNF_ERROR
, "");
1760 TAVOR_TNF_EXIT(tavor_rsrc_fini
);
1764 TAVOR_TNF_EXIT(tavor_rsrc_fini
);
1769 * tavor_rsrc_mbox_init()
1770 * Context: Only called from attach() path context
1773 tavor_rsrc_mbox_init(tavor_state_t
*state
, tavor_rsrc_mbox_info_t
*info
)
1775 tavor_rsrc_pool_info_t
*rsrc_pool
;
1776 tavor_rsrc_priv_mbox_t
*priv
;
1779 uint_t dma_xfer_mode
;
1781 TAVOR_TNF_ENTER(tavor_rsrc_mbox_init
);
1783 ASSERT(state
!= NULL
);
1784 ASSERT(info
!= NULL
);
1786 rsrc_pool
= info
->mbi_rsrcpool
;
1787 ASSERT(rsrc_pool
!= NULL
);
1789 dma_xfer_mode
= state
->ts_cfg_profile
->cp_streaming_consistent
;
1791 /* Allocate and initialize mailbox private structure */
1792 priv
= kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t
), KM_SLEEP
);
1793 priv
->pmb_dip
= state
->ts_dip
;
1794 priv
->pmb_acchdl
= state
->ts_reg_ddrhdl
;
1795 priv
->pmb_devaccattr
= state
->ts_reg_accattr
;
1796 priv
->pmb_xfer_mode
= dma_xfer_mode
;
1799 * Initialize many of the default DMA attributes. Then set alignment
1800 * and scatter-gather restrictions specific for mailbox memory.
1802 tavor_dma_attr_init(&priv
->pmb_dmaattr
);
1803 priv
->pmb_dmaattr
.dma_attr_align
= TAVOR_MBOX_ALIGN
;
1804 priv
->pmb_dmaattr
.dma_attr_sgllen
= 1;
1806 rsrc_pool
->rsrc_private
= priv
;
1808 /* Is object in DDR memory or system memory? */
1809 if (rsrc_pool
->rsrc_loc
== TAVOR_IN_DDR
) {
1810 rsrc_pool
->rsrc_ddr_offset
= vmem_xalloc(state
->ts_ddrvmem
,
1811 rsrc_pool
->rsrc_pool_size
, rsrc_pool
->rsrc_align
,
1812 0, 0, NULL
, NULL
, VM_SLEEP
);
1813 if (rsrc_pool
->rsrc_ddr_offset
== NULL
) {
1814 /* Unable to alloc space for mailboxes */
1815 kmem_free(priv
, sizeof (tavor_rsrc_priv_mbox_t
));
1816 TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail
,
1817 TAVOR_TNF_ERROR
, "");
1818 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init
);
1819 return (DDI_FAILURE
);
1822 /* Calculate offset and starting point (in DDR) */
1823 offset
= ((uintptr_t)rsrc_pool
->rsrc_ddr_offset
-
1824 state
->ts_ddr
.ddr_baseaddr
);
1825 rsrc_pool
->rsrc_start
=
1826 (void *)(uintptr_t)((uintptr_t)state
->ts_reg_ddr_baseaddr
+
1829 /* Create new vmem arena for the mailboxes */
1830 vmp
= vmem_create(info
->mbi_rsrcname
,
1831 rsrc_pool
->rsrc_ddr_offset
, rsrc_pool
->rsrc_pool_size
,
1832 rsrc_pool
->rsrc_quantum
, NULL
, NULL
, NULL
, 0, VM_SLEEP
);
1834 /* Unable to create vmem arena */
1835 vmem_xfree(state
->ts_ddrvmem
,
1836 rsrc_pool
->rsrc_ddr_offset
,
1837 rsrc_pool
->rsrc_pool_size
);
1838 kmem_free(priv
, sizeof (tavor_rsrc_priv_mbox_t
));
1839 TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail
,
1840 TAVOR_TNF_ERROR
, "");
1841 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init
);
1842 return (DDI_FAILURE
);
1844 rsrc_pool
->rsrc_vmp
= vmp
;
1846 rsrc_pool
->rsrc_ddr_offset
= NULL
;
1847 rsrc_pool
->rsrc_start
= NULL
;
1848 rsrc_pool
->rsrc_vmp
= NULL
;
1851 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init
);
1852 return (DDI_SUCCESS
);
1857 * tavor_rsrc_mbox_fini()
1858 * Context: Only called from attach() and/or detach() path contexts
1861 tavor_rsrc_mbox_fini(tavor_state_t
*state
, tavor_rsrc_mbox_info_t
*info
)
1863 tavor_rsrc_pool_info_t
*rsrc_pool
;
1865 TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini
);
1867 ASSERT(state
!= NULL
);
1868 ASSERT(info
!= NULL
);
1870 rsrc_pool
= info
->mbi_rsrcpool
;
1871 ASSERT(rsrc_pool
!= NULL
);
1873 /* If mailboxes are DDR memory, then destroy and free up vmem */
1874 if (rsrc_pool
->rsrc_loc
== TAVOR_IN_DDR
) {
1876 /* Destroy the specially created mbox vmem arena */
1877 vmem_destroy(rsrc_pool
->rsrc_vmp
);
1879 /* Free up the region from the ddr_vmem arena */
1880 vmem_xfree(state
->ts_ddrvmem
, rsrc_pool
->rsrc_ddr_offset
,
1881 rsrc_pool
->rsrc_pool_size
);
1884 /* Free up the private struct */
1885 kmem_free(rsrc_pool
->rsrc_private
, sizeof (tavor_rsrc_priv_mbox_t
));
1887 TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini
);
1892 * tavor_rsrc_hw_entries_init()
1893 * Context: Only called from attach() path context
1896 tavor_rsrc_hw_entries_init(tavor_state_t
*state
,
1897 tavor_rsrc_hw_entry_info_t
*info
)
1899 tavor_rsrc_pool_info_t
*rsrc_pool
;
1900 tavor_rsrc_t
*rsvd_rsrc
= NULL
;
1902 uint64_t num_hwentry
, max_hwentry
, num_prealloc
;
1906 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init
);
1908 ASSERT(state
!= NULL
);
1909 ASSERT(info
!= NULL
);
1911 rsrc_pool
= info
->hwi_rsrcpool
;
1912 ASSERT(rsrc_pool
!= NULL
);
1913 num_hwentry
= info
->hwi_num
;
1914 max_hwentry
= info
->hwi_max
;
1915 num_prealloc
= info
->hwi_prealloc
;
1917 /* Make sure number of HW entries makes sense */
1918 if (num_hwentry
> max_hwentry
) {
1919 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail
,
1920 TAVOR_TNF_ERROR
, "", tnf_string
, errmsg
, "number of HW "
1921 "entries exceeds device maximum", tnf_uint
, maxhw
,
1923 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init
);
1924 return (DDI_FAILURE
);
1928 * Determine if we need to allocate DDR space to set up the
1929 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already
1930 * been initialized (as is the case for the UAR page init).
1932 if (rsrc_pool
->rsrc_start
== NULL
) {
1933 /* Make sure HW entries table is aligned as specified */
1934 rsrc_pool
->rsrc_ddr_offset
= vmem_xalloc(state
->ts_ddrvmem
,
1935 rsrc_pool
->rsrc_pool_size
, rsrc_pool
->rsrc_align
,
1936 0, 0, NULL
, NULL
, VM_NOSLEEP
| VM_FIRSTFIT
);
1937 if (rsrc_pool
->rsrc_ddr_offset
== NULL
) {
1938 /* Unable to alloc space for aligned HW table */
1939 TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail
,
1940 TAVOR_TNF_ERROR
, "");
1941 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init
);
1942 return (DDI_FAILURE
);
1945 /* Calculate offset and starting point (in DDR) */
1946 offset
= ((uintptr_t)rsrc_pool
->rsrc_ddr_offset
-
1947 state
->ts_ddr
.ddr_baseaddr
);
1948 rsrc_pool
->rsrc_start
=
1949 (void *)(uintptr_t)((uintptr_t)state
->ts_reg_ddr_baseaddr
+
1952 rsrc_pool
->rsrc_ddr_offset
= rsrc_pool
->rsrc_start
;
1956 * Create new vmem arena for the HW entries table (if rsrc_quantum
1957 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW
1958 * entries are not going to be dynamically allocatable (i.e. they
1959 * won't be allocated/freed through tavor_rsrc_alloc/free). This
1960 * latter option is used for EQPC and UARSCR resource which are, in
1961 * fact, managed by the Tavor hardware.
1963 if (rsrc_pool
->rsrc_quantum
!= 0) {
1964 vmp
= vmem_create(info
->hwi_rsrcname
,
1965 rsrc_pool
->rsrc_ddr_offset
, rsrc_pool
->rsrc_pool_size
,
1966 rsrc_pool
->rsrc_quantum
, NULL
, NULL
, NULL
, 0, VM_SLEEP
);
1968 /* Unable to create vmem arena */
1969 if (rsrc_pool
->rsrc_ddr_offset
!=
1970 rsrc_pool
->rsrc_start
) {
1971 vmem_xfree(state
->ts_ddrvmem
,
1972 rsrc_pool
->rsrc_ddr_offset
,
1973 rsrc_pool
->rsrc_pool_size
);
1975 TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail
,
1976 TAVOR_TNF_ERROR
, "");
1977 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init
);
1978 return (DDI_FAILURE
);
1980 rsrc_pool
->rsrc_vmp
= vmp
;
1982 rsrc_pool
->rsrc_vmp
= NULL
;
1985 /* The first HW entries may be reserved by Tavor firmware */
1986 if (num_prealloc
!= 0) {
1987 status
= tavor_rsrc_alloc(state
, rsrc_pool
->rsrc_type
,
1988 num_prealloc
, TAVOR_SLEEP
, &rsvd_rsrc
);
1989 if (status
!= DDI_SUCCESS
) {
1990 /* Unable to preallocate the reserved HW entries */
1991 if (rsrc_pool
->rsrc_vmp
!= NULL
) {
1992 vmem_destroy(rsrc_pool
->rsrc_vmp
);
1994 if (rsrc_pool
->rsrc_ddr_offset
!=
1995 rsrc_pool
->rsrc_start
) {
1996 vmem_xfree(state
->ts_ddrvmem
,
1997 rsrc_pool
->rsrc_ddr_offset
,
1998 rsrc_pool
->rsrc_pool_size
);
2000 TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail
,
2001 TAVOR_TNF_ERROR
, "");
2002 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init
);
2003 return (DDI_FAILURE
);
2006 rsrc_pool
->rsrc_private
= rsvd_rsrc
;
2008 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init
);
2009 return (DDI_SUCCESS
);
2014 * tavor_rsrc_hw_entries_fini()
2015 * Context: Only called from attach() and/or detach() path contexts
2018 tavor_rsrc_hw_entries_fini(tavor_state_t
*state
,
2019 tavor_rsrc_hw_entry_info_t
*info
)
2021 tavor_rsrc_pool_info_t
*rsrc_pool
;
2022 tavor_rsrc_t
*rsvd_rsrc
;
2024 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini
);
2026 ASSERT(state
!= NULL
);
2027 ASSERT(info
!= NULL
);
2029 rsrc_pool
= info
->hwi_rsrcpool
;
2030 ASSERT(rsrc_pool
!= NULL
);
2032 /* Free up any "reserved" (i.e. preallocated) HW entries */
2033 rsvd_rsrc
= (tavor_rsrc_t
*)rsrc_pool
->rsrc_private
;
2034 if (rsvd_rsrc
!= NULL
) {
2035 tavor_rsrc_free(state
, &rsvd_rsrc
);
2039 * If we've actually setup a vmem arena for the HW entries, then
2042 if (rsrc_pool
->rsrc_vmp
!= NULL
) {
2043 vmem_destroy(rsrc_pool
->rsrc_vmp
);
2047 * Determine if a region was allocated from the tavor_ddr_vmem
2048 * arena (and free it up if necessary)
2050 if (rsrc_pool
->rsrc_ddr_offset
!= rsrc_pool
->rsrc_start
) {
2051 vmem_xfree(state
->ts_ddrvmem
, rsrc_pool
->rsrc_ddr_offset
,
2052 rsrc_pool
->rsrc_pool_size
);
2055 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini
);
2060 * tavor_rsrc_sw_handles_init()
2061 * Context: Only called from attach() path context
2065 tavor_rsrc_sw_handles_init(tavor_state_t
*state
, tavor_rsrc_sw_hdl_info_t
*info
)
2067 tavor_rsrc_pool_info_t
*rsrc_pool
;
2068 uint64_t num_swhdl
, max_swhdl
, prealloc_sz
;
2070 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init
);
2072 ASSERT(state
!= NULL
);
2073 ASSERT(info
!= NULL
);
2075 rsrc_pool
= info
->swi_rsrcpool
;
2076 ASSERT(rsrc_pool
!= NULL
);
2077 num_swhdl
= info
->swi_num
;
2078 max_swhdl
= info
->swi_max
;
2079 prealloc_sz
= info
->swi_prealloc_sz
;
2081 /* Make sure number of SW handles makes sense */
2082 if (num_swhdl
> max_swhdl
) {
2083 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail
,
2084 TAVOR_TNF_ERROR
, "", tnf_string
, errmsg
, "number of SW "
2085 "handles exceeds maximum", tnf_uint
, maxsw
, max_swhdl
);
2086 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init
);
2087 return (DDI_FAILURE
);
2091 * Depending on the flags parameter, create a kmem_cache for some
2092 * number of software handle structures. Note: kmem_cache_create()
2093 * will SLEEP until successful.
2095 if (info
->swi_flags
& TAVOR_SWHDL_KMEMCACHE_INIT
) {
2096 rsrc_pool
->rsrc_private
= kmem_cache_create(
2097 info
->swi_rsrcname
, rsrc_pool
->rsrc_quantum
, 0,
2098 info
->swi_constructor
, info
->swi_destructor
, NULL
,
2099 rsrc_pool
->rsrc_state
, NULL
, 0);
2102 /* Allocate the central list of SW handle pointers */
2103 if (info
->swi_flags
& TAVOR_SWHDL_TABLE_INIT
) {
2104 info
->swi_table_ptr
= kmem_zalloc(num_swhdl
* prealloc_sz
,
2108 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init
);
2109 return (DDI_SUCCESS
);
2114 * tavor_rsrc_sw_handles_fini()
2115 * Context: Only called from attach() and/or detach() path contexts
2119 tavor_rsrc_sw_handles_fini(tavor_state_t
*state
, tavor_rsrc_sw_hdl_info_t
*info
)
2121 tavor_rsrc_pool_info_t
*rsrc_pool
;
2122 uint64_t num_swhdl
, prealloc_sz
;
2124 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini
);
2126 ASSERT(state
!= NULL
);
2127 ASSERT(info
!= NULL
);
2129 rsrc_pool
= info
->swi_rsrcpool
;
2130 num_swhdl
= info
->swi_num
;
2131 prealloc_sz
= info
->swi_prealloc_sz
;
2134 * If a "software handle" kmem_cache exists for this resource, then
2137 if (rsrc_pool
!= NULL
) {
2138 kmem_cache_destroy(rsrc_pool
->rsrc_private
);
2141 /* Free up this central list of SW handle pointers */
2142 if (info
->swi_table_ptr
!= NULL
) {
2143 kmem_free(info
->swi_table_ptr
, num_swhdl
* prealloc_sz
);
2146 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini
);
2151 * tavor_rsrc_pd_handles_init()
2152 * Context: Only called from attach() path context
2155 tavor_rsrc_pd_handles_init(tavor_state_t
*state
, tavor_rsrc_sw_hdl_info_t
*info
)
2157 tavor_rsrc_pool_info_t
*rsrc_pool
;
2159 char vmem_name
[TAVOR_RSRC_NAME_MAXLEN
];
2162 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init
);
2164 ASSERT(state
!= NULL
);
2165 ASSERT(info
!= NULL
);
2167 rsrc_pool
= info
->swi_rsrcpool
;
2168 ASSERT(rsrc_pool
!= NULL
);
2170 /* Initialize the resource pool for software handle table */
2171 status
= tavor_rsrc_sw_handles_init(state
, info
);
2172 if (status
!= DDI_SUCCESS
) {
2173 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail
, TAVOR_TNF_ERROR
, "");
2174 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc
);
2175 return (DDI_FAILURE
);
2178 /* Build vmem arena name from Tavor instance */
2179 TAVOR_RSRC_NAME(vmem_name
, TAVOR_PDHDL_VMEM
);
2181 /* Create new vmem arena for PD numbers */
2182 vmp
= vmem_create(vmem_name
, (caddr_t
)1, info
->swi_num
, 1, NULL
,
2183 NULL
, NULL
, 0, VM_SLEEP
| VMC_IDENTIFIER
);
2185 /* Unable to create vmem arena */
2186 info
->swi_table_ptr
= NULL
;
2187 tavor_rsrc_sw_handles_fini(state
, info
);
2188 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail
,
2189 TAVOR_TNF_ERROR
, "");
2190 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init
);
2191 return (DDI_FAILURE
);
2193 rsrc_pool
->rsrc_vmp
= vmp
;
2195 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init
);
2196 return (DDI_SUCCESS
);
2201 * tavor_rsrc_pd_handles_fini()
2202 * Context: Only called from attach() and/or detach() path contexts
2205 tavor_rsrc_pd_handles_fini(tavor_state_t
*state
, tavor_rsrc_sw_hdl_info_t
*info
)
2207 tavor_rsrc_pool_info_t
*rsrc_pool
;
2209 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini
);
2211 ASSERT(state
!= NULL
);
2212 ASSERT(info
!= NULL
);
2214 rsrc_pool
= info
->swi_rsrcpool
;
2216 /* Destroy the specially created UAR scratch table vmem arena */
2217 vmem_destroy(rsrc_pool
->rsrc_vmp
);
2219 /* Destroy the "tavor_sw_pd_t" kmem_cache */
2220 tavor_rsrc_sw_handles_fini(state
, info
);
2222 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini
);
2227 * tavor_rsrc_mbox_alloc()
2228 * Context: Only called from attach() path context
2231 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t
*pool_info
, uint_t num
,
2234 tavor_rsrc_priv_mbox_t
*priv
;
2238 size_t real_len
, temp_len
;
2241 TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc
);
2243 ASSERT(pool_info
!= NULL
);
2244 ASSERT(hdl
!= NULL
);
2246 /* Get the private pointer for the mailboxes */
2247 priv
= pool_info
->rsrc_private
;
2248 ASSERT(priv
!= NULL
);
2251 * Allocate a DMA handle for the mailbox. This will be used for
2252 * two purposes (potentially). First, it could be used below in
2253 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
2254 * system memory. Second, it is definitely used later to bind
2255 * the mailbox for DMA access from/by the hardware.
2257 status
= ddi_dma_alloc_handle(priv
->pmb_dip
, &priv
->pmb_dmaattr
,
2258 DDI_DMA_SLEEP
, NULL
, &hdl
->tr_dmahdl
);
2259 if (status
!= DDI_SUCCESS
) {
2260 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail
, TAVOR_TNF_ERROR
,
2261 "", tnf_uint
, status
, status
);
2262 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc
);
2263 return (DDI_FAILURE
);
2266 /* Is mailbox in DDR memory or system memory? */
2267 if (pool_info
->rsrc_loc
== TAVOR_IN_DDR
) {
2268 /* Use vmem_alloc() to get DDR address of mbox */
2269 hdl
->tr_len
= (num
* pool_info
->rsrc_quantum
);
2270 addr
= vmem_alloc(pool_info
->rsrc_vmp
, hdl
->tr_len
,
2273 /* No more DDR available for mailbox entries */
2274 ddi_dma_free_handle(&hdl
->tr_dmahdl
);
2275 TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail
,
2276 TAVOR_TNF_ERROR
, "");
2277 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc
);
2278 return (DDI_FAILURE
);
2280 hdl
->tr_acchdl
= priv
->pmb_acchdl
;
2282 /* Calculate kernel virtual address (from the DDR offset) */
2283 offset
= ((uintptr_t)addr
-
2284 (uintptr_t)pool_info
->rsrc_ddr_offset
);
2285 hdl
->tr_addr
= (void *)(uintptr_t)(offset
+
2286 (uintptr_t)pool_info
->rsrc_start
);
2288 } else { /* TAVOR_IN_SYSMEM */
2290 /* Use ddi_dma_mem_alloc() to get memory for mailbox */
2291 temp_len
= (num
* pool_info
->rsrc_quantum
);
2292 status
= ddi_dma_mem_alloc(hdl
->tr_dmahdl
, temp_len
,
2293 &priv
->pmb_devaccattr
, priv
->pmb_xfer_mode
, DDI_DMA_SLEEP
,
2294 NULL
, &kaddr
, &real_len
, &hdl
->tr_acchdl
);
2295 if (status
!= DDI_SUCCESS
) {
2296 /* No more sys memory available for mailbox entries */
2297 ddi_dma_free_handle(&hdl
->tr_dmahdl
);
2298 TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail
,
2299 TAVOR_TNF_ERROR
, "");
2300 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc
);
2301 return (DDI_FAILURE
);
2303 hdl
->tr_addr
= (void *)kaddr
;
2304 hdl
->tr_len
= real_len
;
2307 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc
);
2308 return (DDI_SUCCESS
);
2313 * tavor_rsrc_mbox_free()
2314 * Context: Can be called from interrupt or base context.
2317 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t
*pool_info
, tavor_rsrc_t
*hdl
)
2322 TAVOR_TNF_ENTER(tavor_rsrc_mbox_free
);
2324 ASSERT(pool_info
!= NULL
);
2325 ASSERT(hdl
!= NULL
);
2327 /* Is mailbox in DDR memory or system memory? */
2328 if (pool_info
->rsrc_loc
== TAVOR_IN_DDR
) {
2330 /* Calculate the allocated address (the mbox's DDR offset) */
2331 offset
= ((uintptr_t)hdl
->tr_addr
-
2332 (uintptr_t)pool_info
->rsrc_start
);
2333 addr
= (void *)(uintptr_t)(offset
+
2334 (uintptr_t)pool_info
->rsrc_ddr_offset
);
2336 /* Use vmem_free() to free up DDR memory for mailbox */
2337 vmem_free(pool_info
->rsrc_vmp
, addr
, hdl
->tr_len
);
2339 } else { /* TAVOR_IN_SYSMEM */
2341 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
2342 ddi_dma_mem_free(&hdl
->tr_acchdl
);
2345 /* Free the DMA handle for the mailbox */
2346 ddi_dma_free_handle(&hdl
->tr_dmahdl
);
2348 TAVOR_TNF_EXIT(tavor_rsrc_mbox_free
);
2353 * tavor_rsrc_hw_entry_alloc()
2354 * Context: Can be called from interrupt or base context.
2357 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t
*pool_info
, uint_t num
,
2358 uint_t num_align
, ddi_acc_handle_t acc_handle
, uint_t sleepflag
,
2366 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc
);
2368 ASSERT(pool_info
!= NULL
);
2369 ASSERT(hdl
!= NULL
);
2372 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
2373 * use dma_handle (because they are in Tavor locally attached DDR
2374 * memory) and, generally, don't use the acc_handle (because the
2375 * entries are not directly accessed by software). The exceptions
2376 * to this rule are the UARPG and UDAV entries.
2380 * Use vmem_xalloc() to get a properly aligned pointer (based on
2381 * the number requested) to the HW entry(ies). This handles the
2382 * cases (for special QPCs and for RDB entries) where we need more
2383 * than one and need to ensure that they are properly aligned.
2385 flag
= (sleepflag
== TAVOR_SLEEP
) ? VM_SLEEP
: VM_NOSLEEP
;
2386 hdl
->tr_len
= (num
* pool_info
->rsrc_quantum
);
2387 align
= (num_align
* pool_info
->rsrc_quantum
);
2388 addr
= vmem_xalloc(pool_info
->rsrc_vmp
, hdl
->tr_len
,
2389 align
, 0, 0, NULL
, NULL
, flag
| VM_FIRSTFIT
);
2391 /* No more HW entries available */
2392 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail
,
2393 TAVOR_TNF_ERROR
, "");
2394 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc
);
2395 return (DDI_FAILURE
);
2398 /* If an access handle was provided, fill it in */
2399 if (acc_handle
!= 0) {
2400 hdl
->tr_acchdl
= acc_handle
;
2403 /* Calculate vaddr and HW table index (from the DDR offset) */
2404 offset
= ((uintptr_t)addr
- (uintptr_t)pool_info
->rsrc_ddr_offset
);
2405 hdl
->tr_addr
= (void *)(uintptr_t)(offset
+
2406 (uintptr_t)pool_info
->rsrc_start
);
2407 hdl
->tr_indx
= (offset
>> pool_info
->rsrc_shift
);
2409 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc
);
2410 return (DDI_SUCCESS
);
2415 * tavor_rsrc_hw_entry_free()
2416 * Context: Can be called from interrupt or base context.
2419 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t
*pool_info
, tavor_rsrc_t
*hdl
)
2424 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free
);
2426 ASSERT(pool_info
!= NULL
);
2427 ASSERT(hdl
!= NULL
);
2429 /* Calculate the allocated address (the entry's DDR offset) */
2430 offset
= ((uintptr_t)hdl
->tr_addr
- (uintptr_t)pool_info
->rsrc_start
);
2431 addr
= (void *)(uintptr_t)(offset
+
2432 (uintptr_t)pool_info
->rsrc_ddr_offset
);
2434 /* Use vmem_xfree() to free up the HW table entry */
2435 vmem_xfree(pool_info
->rsrc_vmp
, addr
, hdl
->tr_len
);
2437 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free
);
2442 * tavor_rsrc_swhdl_alloc()
2443 * Context: Can be called from interrupt or base context.
2446 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t
*pool_info
, uint_t sleepflag
,
2452 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc
);
2454 ASSERT(pool_info
!= NULL
);
2455 ASSERT(hdl
!= NULL
);
2457 /* Allocate the software handle structure */
2458 flag
= (sleepflag
== TAVOR_SLEEP
) ? KM_SLEEP
: KM_NOSLEEP
;
2459 addr
= kmem_cache_alloc(pool_info
->rsrc_private
, flag
);
2461 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail
, TAVOR_TNF_ERROR
,
2463 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc
);
2464 return (DDI_FAILURE
);
2466 hdl
->tr_len
= pool_info
->rsrc_quantum
;
2467 hdl
->tr_addr
= addr
;
2469 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc
);
2470 return (DDI_SUCCESS
);
2475 * tavor_rsrc_swhdl_free()
2476 * Context: Can be called from interrupt or base context.
2479 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t
*pool_info
, tavor_rsrc_t
*hdl
)
2481 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free
);
2483 ASSERT(pool_info
!= NULL
);
2484 ASSERT(hdl
!= NULL
);
2486 /* Free the software handle structure */
2487 kmem_cache_free(pool_info
->rsrc_private
, hdl
->tr_addr
);
2489 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free
);
2494 * tavor_rsrc_pdhdl_alloc()
2495 * Context: Can be called from interrupt or base context.
2498 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t
*pool_info
, uint_t sleepflag
,
2505 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc
);
2507 ASSERT(pool_info
!= NULL
);
2508 ASSERT(hdl
!= NULL
);
2510 /* Allocate the software handle */
2511 status
= tavor_rsrc_swhdl_alloc(pool_info
, sleepflag
, hdl
);
2512 if (status
!= DDI_SUCCESS
) {
2513 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail
, TAVOR_TNF_ERROR
, "");
2514 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc
);
2515 return (DDI_FAILURE
);
2517 addr
= (tavor_pdhdl_t
)hdl
->tr_addr
;
2518 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr
))
2520 /* Allocate a PD number for the handle */
2521 flag
= (sleepflag
== TAVOR_SLEEP
) ? VM_SLEEP
: VM_NOSLEEP
;
2522 tmpaddr
= vmem_alloc(pool_info
->rsrc_vmp
, 1, flag
);
2523 if (tmpaddr
== NULL
) {
2524 /* No more PD number entries available */
2525 tavor_rsrc_swhdl_free(pool_info
, hdl
);
2526 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail
,
2527 TAVOR_TNF_ERROR
, "");
2528 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc
);
2529 return (DDI_FAILURE
);
2531 addr
->pd_pdnum
= (uint32_t)(uintptr_t)tmpaddr
;
2532 addr
->pd_rsrcp
= hdl
;
2533 hdl
->tr_indx
= addr
->pd_pdnum
;
2535 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc
);
2536 return (DDI_SUCCESS
);
2541 * tavor_rsrc_pdhdl_free()
2542 * Context: Can be called from interrupt or base context.
2545 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t
*pool_info
, tavor_rsrc_t
*hdl
)
2547 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free
);
2549 ASSERT(pool_info
!= NULL
);
2550 ASSERT(hdl
!= NULL
);
2552 /* Use vmem_free() to free up the PD number */
2553 vmem_free(pool_info
->rsrc_vmp
, (void *)(uintptr_t)hdl
->tr_indx
, 1);
2555 /* Free the software handle structure */
2556 tavor_rsrc_swhdl_free(pool_info
, hdl
);
2558 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free
);
2563 * tavor_rsrc_pdhdl_constructor()
2564 * Context: Can be called from interrupt or base context.
2568 tavor_rsrc_pdhdl_constructor(void *pd
, void *priv
, int flags
)
2570 tavor_pdhdl_t pdhdl
;
2571 tavor_state_t
*state
;
2573 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor
);
2575 pdhdl
= (tavor_pdhdl_t
)pd
;
2576 state
= (tavor_state_t
*)priv
;
2578 mutex_init(&pdhdl
->pd_lock
, NULL
, MUTEX_DRIVER
,
2579 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2581 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor
);
2582 return (DDI_SUCCESS
);
2587 * tavor_rsrc_pdhdl_destructor()
2588 * Context: Can be called from interrupt or base context.
2592 tavor_rsrc_pdhdl_destructor(void *pd
, void *priv
)
2594 tavor_pdhdl_t pdhdl
;
2596 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor
);
2598 pdhdl
= (tavor_pdhdl_t
)pd
;
2600 mutex_destroy(&pdhdl
->pd_lock
);
2602 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor
);
2607 * tavor_rsrc_cqhdl_constructor()
2608 * Context: Can be called from interrupt or base context.
2612 tavor_rsrc_cqhdl_constructor(void *cq
, void *priv
, int flags
)
2614 tavor_cqhdl_t cqhdl
;
2615 tavor_state_t
*state
;
2617 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor
);
2619 cqhdl
= (tavor_cqhdl_t
)cq
;
2620 state
= (tavor_state_t
*)priv
;
2622 mutex_init(&cqhdl
->cq_lock
, NULL
, MUTEX_DRIVER
,
2623 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2624 mutex_init(&cqhdl
->cq_wrid_wqhdr_lock
, NULL
, MUTEX_DRIVER
,
2625 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2627 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor
);
2628 return (DDI_SUCCESS
);
2633 * tavor_rsrc_cqhdl_destructor()
2634 * Context: Can be called from interrupt or base context.
2638 tavor_rsrc_cqhdl_destructor(void *cq
, void *priv
)
2640 tavor_cqhdl_t cqhdl
;
2642 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor
);
2644 cqhdl
= (tavor_cqhdl_t
)cq
;
2646 mutex_destroy(&cqhdl
->cq_wrid_wqhdr_lock
);
2647 mutex_destroy(&cqhdl
->cq_lock
);
2649 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor
);
2654 * tavor_rsrc_qphdl_constructor()
2655 * Context: Can be called from interrupt or base context.
2659 tavor_rsrc_qphdl_constructor(void *qp
, void *priv
, int flags
)
2661 tavor_qphdl_t qphdl
;
2662 tavor_state_t
*state
;
2664 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor
);
2666 qphdl
= (tavor_qphdl_t
)qp
;
2667 state
= (tavor_state_t
*)priv
;
2669 mutex_init(&qphdl
->qp_lock
, NULL
, MUTEX_DRIVER
,
2670 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2672 TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor
);
2673 return (DDI_SUCCESS
);
2678 * tavor_rsrc_qphdl_destructor()
2679 * Context: Can be called from interrupt or base context.
2683 tavor_rsrc_qphdl_destructor(void *qp
, void *priv
)
2685 tavor_qphdl_t qphdl
;
2687 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor
);
2689 qphdl
= (tavor_qphdl_t
)qp
;
2691 mutex_destroy(&qphdl
->qp_lock
);
2693 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor
);
2698 * tavor_rsrc_srqhdl_constructor()
2699 * Context: Can be called from interrupt or base context.
2703 tavor_rsrc_srqhdl_constructor(void *srq
, void *priv
, int flags
)
2705 tavor_srqhdl_t srqhdl
;
2706 tavor_state_t
*state
;
2708 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor
);
2710 srqhdl
= (tavor_srqhdl_t
)srq
;
2711 state
= (tavor_state_t
*)priv
;
2713 mutex_init(&srqhdl
->srq_lock
, NULL
, MUTEX_DRIVER
,
2714 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2716 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor
);
2717 return (DDI_SUCCESS
);
2722 * tavor_rsrc_srqhdl_destructor()
2723 * Context: Can be called from interrupt or base context.
2727 tavor_rsrc_srqhdl_destructor(void *srq
, void *priv
)
2729 tavor_srqhdl_t srqhdl
;
2731 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor
);
2733 srqhdl
= (tavor_srqhdl_t
)srq
;
2735 mutex_destroy(&srqhdl
->srq_lock
);
2737 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor
);
2742 * tavor_rsrc_refcnt_constructor()
2743 * Context: Can be called from interrupt or base context.
2747 tavor_rsrc_refcnt_constructor(void *rc
, void *priv
, int flags
)
2749 tavor_sw_refcnt_t
*refcnt
;
2750 tavor_state_t
*state
;
2752 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor
);
2754 refcnt
= (tavor_sw_refcnt_t
*)rc
;
2755 state
= (tavor_state_t
*)priv
;
2757 mutex_init(&refcnt
->swrc_lock
, NULL
, MUTEX_DRIVER
,
2758 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2760 TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor
);
2761 return (DDI_SUCCESS
);
2766 * tavor_rsrc_refcnt_destructor()
2767 * Context: Can be called from interrupt or base context.
2771 tavor_rsrc_refcnt_destructor(void *rc
, void *priv
)
2773 tavor_sw_refcnt_t
*refcnt
;
2775 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor
);
2777 refcnt
= (tavor_sw_refcnt_t
*)rc
;
2779 mutex_destroy(&refcnt
->swrc_lock
);
2781 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor
);
2786 * tavor_rsrc_ahhdl_constructor()
2787 * Context: Can be called from interrupt or base context.
2791 tavor_rsrc_ahhdl_constructor(void *ah
, void *priv
, int flags
)
2793 tavor_ahhdl_t ahhdl
;
2794 tavor_state_t
*state
;
2796 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor
);
2798 ahhdl
= (tavor_ahhdl_t
)ah
;
2799 state
= (tavor_state_t
*)priv
;
2801 mutex_init(&ahhdl
->ah_lock
, NULL
, MUTEX_DRIVER
,
2802 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2804 TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor
);
2805 return (DDI_SUCCESS
);
2810 * tavor_rsrc_ahhdl_destructor()
2811 * Context: Can be called from interrupt or base context.
2815 tavor_rsrc_ahhdl_destructor(void *ah
, void *priv
)
2817 tavor_ahhdl_t ahhdl
;
2819 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor
);
2821 ahhdl
= (tavor_ahhdl_t
)ah
;
2823 mutex_destroy(&ahhdl
->ah_lock
);
2825 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor
);
2830 * tavor_rsrc_mrhdl_constructor()
2831 * Context: Can be called from interrupt or base context.
2835 tavor_rsrc_mrhdl_constructor(void *mr
, void *priv
, int flags
)
2837 tavor_mrhdl_t mrhdl
;
2838 tavor_state_t
*state
;
2840 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor
);
2842 mrhdl
= (tavor_mrhdl_t
)mr
;
2843 state
= (tavor_state_t
*)priv
;
2845 mutex_init(&mrhdl
->mr_lock
, NULL
, MUTEX_DRIVER
,
2846 DDI_INTR_PRI(state
->ts_intrmsi_pri
));
2848 TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor
);
2849 return (DDI_SUCCESS
);
2854 * tavor_rsrc_mrhdl_destructor()
2855 * Context: Can be called from interrupt or base context.
2859 tavor_rsrc_mrhdl_destructor(void *mr
, void *priv
)
2861 tavor_mrhdl_t mrhdl
;
2863 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor
);
2865 mrhdl
= (tavor_mrhdl_t
)mr
;
2867 mutex_destroy(&mrhdl
->mr_lock
);
2869 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor
);
2874 * tavor_rsrc_mcg_entry_get_size()
2877 tavor_rsrc_mcg_entry_get_size(tavor_state_t
*state
, uint_t
*mcg_size_shift
)
2879 uint_t num_qp_per_mcg
, max_qp_per_mcg
, log2
;
2881 TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size
);
2884 * Round the configured number of QP per MCG to next larger
2885 * power-of-2 size and update.
2887 num_qp_per_mcg
= state
->ts_cfg_profile
->cp_num_qp_per_mcg
+ 8;
2888 log2
= highbit(num_qp_per_mcg
);
2889 if (ISP2(num_qp_per_mcg
)) {
2892 state
->ts_cfg_profile
->cp_num_qp_per_mcg
= (1 << log2
) - 8;
2894 /* Now make sure number of QP per MCG makes sense */
2895 num_qp_per_mcg
= state
->ts_cfg_profile
->cp_num_qp_per_mcg
;
2896 max_qp_per_mcg
= (1 << state
->ts_devlim
.log_max_qp_mcg
);
2897 if (num_qp_per_mcg
> max_qp_per_mcg
) {
2898 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail
,
2899 TAVOR_TNF_ERROR
, "", tnf_uint
, maxqpmcg
, max_qp_per_mcg
);
2900 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size
);
2901 return (DDI_FAILURE
);
2904 /* Return the (shift) size of an individual MCG HW entry */
2905 *mcg_size_shift
= log2
+ 2;
2907 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size
);
2908 return (DDI_SUCCESS
);