2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * $Id: mthca_profile.c 1349 2004-12-16 21:09:43Z roland $
36 #include <linux/module.h>
37 #include <linux/moduleparam.h>
39 #include "mthca_profile.h"
61 MTHCA_NUM_PDS
= 1 << 15
64 u64
mthca_make_profile(struct mthca_dev
*dev
,
65 struct mthca_profile
*request
,
66 struct mthca_dev_lim
*dev_lim
,
67 struct mthca_init_hca_param
*init_hca
)
69 struct mthca_resource
{
77 u64 mem_base
, mem_avail
;
79 struct mthca_resource
*profile
;
80 struct mthca_resource tmp
;
83 profile
= kmalloc(MTHCA_RES_NUM
* sizeof *profile
, GFP_KERNEL
);
87 memset(profile
, 0, MTHCA_RES_NUM
* sizeof *profile
);
89 profile
[MTHCA_RES_QP
].size
= dev_lim
->qpc_entry_sz
;
90 profile
[MTHCA_RES_EEC
].size
= dev_lim
->eec_entry_sz
;
91 profile
[MTHCA_RES_SRQ
].size
= dev_lim
->srq_entry_sz
;
92 profile
[MTHCA_RES_CQ
].size
= dev_lim
->cqc_entry_sz
;
93 profile
[MTHCA_RES_EQP
].size
= dev_lim
->eqpc_entry_sz
;
94 profile
[MTHCA_RES_EEEC
].size
= dev_lim
->eeec_entry_sz
;
95 profile
[MTHCA_RES_EQ
].size
= dev_lim
->eqc_entry_sz
;
96 profile
[MTHCA_RES_RDB
].size
= MTHCA_RDB_ENTRY_SIZE
;
97 profile
[MTHCA_RES_MCG
].size
= MTHCA_MGM_ENTRY_SIZE
;
98 profile
[MTHCA_RES_MPT
].size
= dev_lim
->mpt_entry_sz
;
99 profile
[MTHCA_RES_MTT
].size
= MTHCA_MTT_SEG_SIZE
;
100 profile
[MTHCA_RES_UAR
].size
= dev_lim
->uar_scratch_entry_sz
;
101 profile
[MTHCA_RES_UDAV
].size
= MTHCA_AV_SIZE
;
102 profile
[MTHCA_RES_UARC
].size
= request
->uarc_size
;
104 profile
[MTHCA_RES_QP
].num
= request
->num_qp
;
105 profile
[MTHCA_RES_EQP
].num
= request
->num_qp
;
106 profile
[MTHCA_RES_RDB
].num
= request
->num_qp
* request
->rdb_per_qp
;
107 profile
[MTHCA_RES_CQ
].num
= request
->num_cq
;
108 profile
[MTHCA_RES_EQ
].num
= MTHCA_NUM_EQS
;
109 profile
[MTHCA_RES_MCG
].num
= request
->num_mcg
;
110 profile
[MTHCA_RES_MPT
].num
= request
->num_mpt
;
111 profile
[MTHCA_RES_MTT
].num
= request
->num_mtt
;
112 profile
[MTHCA_RES_UAR
].num
= request
->num_uar
;
113 profile
[MTHCA_RES_UARC
].num
= request
->num_uar
;
114 profile
[MTHCA_RES_UDAV
].num
= request
->num_udav
;
116 for (i
= 0; i
< MTHCA_RES_NUM
; ++i
) {
118 profile
[i
].log_num
= max(ffs(profile
[i
].num
) - 1, 0);
119 profile
[i
].size
*= profile
[i
].num
;
120 if (mthca_is_memfree(dev
))
121 profile
[i
].size
= max(profile
[i
].size
, (u64
) PAGE_SIZE
);
124 if (mthca_is_memfree(dev
)) {
126 mem_avail
= dev_lim
->hca
.arbel
.max_icm_sz
;
128 mem_base
= dev
->ddr_start
;
129 mem_avail
= dev
->fw
.tavor
.fw_start
- dev
->ddr_start
;
133 * Sort the resources in decreasing order of size. Since they
134 * all have sizes that are powers of 2, we'll be able to keep
135 * resources aligned to their size and pack them without gaps
136 * using the sorted order.
138 for (i
= MTHCA_RES_NUM
; i
> 0; --i
)
139 for (j
= 1; j
< i
; ++j
) {
140 if (profile
[j
].size
> profile
[j
- 1].size
) {
142 profile
[j
] = profile
[j
- 1];
143 profile
[j
- 1] = tmp
;
147 for (i
= 0; i
< MTHCA_RES_NUM
; ++i
) {
148 if (profile
[i
].size
) {
149 profile
[i
].start
= mem_base
+ total_size
;
150 total_size
+= profile
[i
].size
;
152 if (total_size
> mem_avail
) {
153 mthca_err(dev
, "Profile requires 0x%llx bytes; "
154 "won't in 0x%llx bytes of context memory.\n",
155 (unsigned long long) total_size
,
156 (unsigned long long) mem_avail
);
162 mthca_dbg(dev
, "profile[%2d]--%2d/%2d @ 0x%16llx "
164 i
, profile
[i
].type
, profile
[i
].log_num
,
165 (unsigned long long) profile
[i
].start
,
166 (unsigned long long) profile
[i
].size
);
169 if (mthca_is_memfree(dev
))
170 mthca_dbg(dev
, "HCA context memory: reserving %d KB\n",
171 (int) (total_size
>> 10));
173 mthca_dbg(dev
, "HCA memory: allocated %d KB/%d KB (%d KB free)\n",
174 (int) (total_size
>> 10), (int) (mem_avail
>> 10),
175 (int) ((mem_avail
- total_size
) >> 10));
177 for (i
= 0; i
< MTHCA_RES_NUM
; ++i
) {
178 switch (profile
[i
].type
) {
180 dev
->limits
.num_qps
= profile
[i
].num
;
181 init_hca
->qpc_base
= profile
[i
].start
;
182 init_hca
->log_num_qps
= profile
[i
].log_num
;
185 dev
->limits
.num_eecs
= profile
[i
].num
;
186 init_hca
->eec_base
= profile
[i
].start
;
187 init_hca
->log_num_eecs
= profile
[i
].log_num
;
190 dev
->limits
.num_srqs
= profile
[i
].num
;
191 init_hca
->srqc_base
= profile
[i
].start
;
192 init_hca
->log_num_srqs
= profile
[i
].log_num
;
195 dev
->limits
.num_cqs
= profile
[i
].num
;
196 init_hca
->cqc_base
= profile
[i
].start
;
197 init_hca
->log_num_cqs
= profile
[i
].log_num
;
200 init_hca
->eqpc_base
= profile
[i
].start
;
203 init_hca
->eeec_base
= profile
[i
].start
;
206 dev
->limits
.num_eqs
= profile
[i
].num
;
207 init_hca
->eqc_base
= profile
[i
].start
;
208 init_hca
->log_num_eqs
= profile
[i
].log_num
;
211 for (dev
->qp_table
.rdb_shift
= 0;
212 request
->num_qp
<< dev
->qp_table
.rdb_shift
< profile
[i
].num
;
213 ++dev
->qp_table
.rdb_shift
)
215 dev
->qp_table
.rdb_base
= (u32
) profile
[i
].start
;
216 init_hca
->rdb_base
= profile
[i
].start
;
219 dev
->limits
.num_mgms
= profile
[i
].num
>> 1;
220 dev
->limits
.num_amgms
= profile
[i
].num
>> 1;
221 init_hca
->mc_base
= profile
[i
].start
;
222 init_hca
->log_mc_entry_sz
= ffs(MTHCA_MGM_ENTRY_SIZE
) - 1;
223 init_hca
->log_mc_table_sz
= profile
[i
].log_num
;
224 init_hca
->mc_hash_sz
= 1 << (profile
[i
].log_num
- 1);
227 dev
->limits
.num_mpts
= profile
[i
].num
;
228 dev
->mr_table
.mpt_base
= profile
[i
].start
;
229 init_hca
->mpt_base
= profile
[i
].start
;
230 init_hca
->log_mpt_sz
= profile
[i
].log_num
;
233 dev
->limits
.num_mtt_segs
= profile
[i
].num
;
234 dev
->mr_table
.mtt_base
= profile
[i
].start
;
235 init_hca
->mtt_base
= profile
[i
].start
;
236 init_hca
->mtt_seg_sz
= ffs(MTHCA_MTT_SEG_SIZE
) - 7;
239 dev
->limits
.num_uars
= profile
[i
].num
;
240 init_hca
->uar_scratch_base
= profile
[i
].start
;
243 dev
->av_table
.ddr_av_base
= profile
[i
].start
;
244 dev
->av_table
.num_ddr_avs
= profile
[i
].num
;
247 dev
->uar_table
.uarc_size
= request
->uarc_size
;
248 dev
->uar_table
.uarc_base
= profile
[i
].start
;
249 init_hca
->uarc_base
= profile
[i
].start
;
250 init_hca
->log_uarc_sz
= ffs(request
->uarc_size
) - 13;
251 init_hca
->log_uar_sz
= ffs(request
->num_uar
) - 1;
259 * PDs don't take any HCA memory, but we assign them as part
260 * of the HCA profile anyway.
262 dev
->limits
.num_pds
= MTHCA_NUM_PDS
;
265 * For Tavor, FMRs use ioremapped PCI memory. For 32 bit
266 * systems it may use too much vmalloc space to map all MTT
267 * memory, so we reserve some MTTs for FMR access, taking them
268 * out of the MR pool. They don't use additional memory, but
269 * we assign them as part of the HCA profile anyway.
271 if (mthca_is_memfree(dev
))
272 dev
->limits
.fmr_reserved_mtts
= 0;
274 dev
->limits
.fmr_reserved_mtts
= request
->fmr_reserved_mtts
;