2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/errno.h>
35 #include <linux/slab.h>
37 #include <linux/bitmap.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/vmalloc.h>
43 u32
mlx4_bitmap_alloc(struct mlx4_bitmap
*bitmap
)
47 spin_lock(&bitmap
->lock
);
49 obj
= find_next_zero_bit(bitmap
->table
, bitmap
->max
, bitmap
->last
);
50 if (obj
>= bitmap
->max
) {
51 bitmap
->top
= (bitmap
->top
+ bitmap
->max
+ bitmap
->reserved_top
)
53 obj
= find_first_zero_bit(bitmap
->table
, bitmap
->max
);
56 if (obj
< bitmap
->max
) {
57 set_bit(obj
, bitmap
->table
);
58 bitmap
->last
= (obj
+ 1);
59 if (bitmap
->last
== bitmap
->max
)
65 spin_unlock(&bitmap
->lock
);
70 void mlx4_bitmap_free(struct mlx4_bitmap
*bitmap
, u32 obj
)
72 mlx4_bitmap_free_range(bitmap
, obj
, 1);
75 u32
mlx4_bitmap_alloc_range(struct mlx4_bitmap
*bitmap
, int cnt
, int align
)
79 if (likely(cnt
== 1 && align
== 1))
80 return mlx4_bitmap_alloc(bitmap
);
82 spin_lock(&bitmap
->lock
);
84 obj
= bitmap_find_next_zero_area(bitmap
->table
, bitmap
->max
,
85 bitmap
->last
, cnt
, align
- 1);
86 if (obj
>= bitmap
->max
) {
87 bitmap
->top
= (bitmap
->top
+ bitmap
->max
+ bitmap
->reserved_top
)
89 obj
= bitmap_find_next_zero_area(bitmap
->table
, bitmap
->max
,
93 if (obj
< bitmap
->max
) {
94 bitmap_set(bitmap
->table
, obj
, cnt
);
95 if (obj
== bitmap
->last
) {
96 bitmap
->last
= (obj
+ cnt
);
97 if (bitmap
->last
>= bitmap
->max
)
104 spin_unlock(&bitmap
->lock
);
109 void mlx4_bitmap_free_range(struct mlx4_bitmap
*bitmap
, u32 obj
, int cnt
)
111 obj
&= bitmap
->max
+ bitmap
->reserved_top
- 1;
113 spin_lock(&bitmap
->lock
);
114 bitmap_clear(bitmap
->table
, obj
, cnt
);
115 bitmap
->last
= min(bitmap
->last
, obj
);
116 bitmap
->top
= (bitmap
->top
+ bitmap
->max
+ bitmap
->reserved_top
)
118 spin_unlock(&bitmap
->lock
);
121 int mlx4_bitmap_init(struct mlx4_bitmap
*bitmap
, u32 num
, u32 mask
,
122 u32 reserved_bot
, u32 reserved_top
)
124 /* num must be a power of 2 */
125 if (num
!= roundup_pow_of_two(num
))
130 bitmap
->max
= num
- reserved_top
;
132 bitmap
->reserved_top
= reserved_top
;
133 spin_lock_init(&bitmap
->lock
);
134 bitmap
->table
= kzalloc(BITS_TO_LONGS(bitmap
->max
) *
135 sizeof (long), GFP_KERNEL
);
139 bitmap_set(bitmap
->table
, 0, reserved_bot
);
144 void mlx4_bitmap_cleanup(struct mlx4_bitmap
*bitmap
)
146 kfree(bitmap
->table
);
150 * Handling for queue buffers -- we allocate a bunch of memory and
151 * register it in a memory region at HCA virtual address 0. If the
152 * requested size is > max_direct, we split the allocation into
153 * multiple pages, so we don't require too much contiguous memory.
156 int mlx4_buf_alloc(struct mlx4_dev
*dev
, int size
, int max_direct
,
157 struct mlx4_buf
*buf
)
161 if (size
<= max_direct
) {
164 buf
->page_shift
= get_order(size
) + PAGE_SHIFT
;
165 buf
->direct
.buf
= dma_alloc_coherent(&dev
->pdev
->dev
,
166 size
, &t
, GFP_KERNEL
);
167 if (!buf
->direct
.buf
)
172 while (t
& ((1 << buf
->page_shift
) - 1)) {
177 memset(buf
->direct
.buf
, 0, size
);
181 buf
->nbufs
= (size
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
182 buf
->npages
= buf
->nbufs
;
183 buf
->page_shift
= PAGE_SHIFT
;
184 buf
->page_list
= kcalloc(buf
->nbufs
, sizeof(*buf
->page_list
),
189 for (i
= 0; i
< buf
->nbufs
; ++i
) {
190 buf
->page_list
[i
].buf
=
191 dma_alloc_coherent(&dev
->pdev
->dev
, PAGE_SIZE
,
193 if (!buf
->page_list
[i
].buf
)
196 buf
->page_list
[i
].map
= t
;
198 memset(buf
->page_list
[i
].buf
, 0, PAGE_SIZE
);
201 if (BITS_PER_LONG
== 64) {
203 pages
= kmalloc(sizeof *pages
* buf
->nbufs
, GFP_KERNEL
);
206 for (i
= 0; i
< buf
->nbufs
; ++i
)
207 pages
[i
] = virt_to_page(buf
->page_list
[i
].buf
);
208 buf
->direct
.buf
= vmap(pages
, buf
->nbufs
, VM_MAP
, PAGE_KERNEL
);
210 if (!buf
->direct
.buf
)
218 mlx4_buf_free(dev
, size
, buf
);
222 EXPORT_SYMBOL_GPL(mlx4_buf_alloc
);
224 void mlx4_buf_free(struct mlx4_dev
*dev
, int size
, struct mlx4_buf
*buf
)
229 dma_free_coherent(&dev
->pdev
->dev
, size
, buf
->direct
.buf
,
232 if (BITS_PER_LONG
== 64)
233 vunmap(buf
->direct
.buf
);
235 for (i
= 0; i
< buf
->nbufs
; ++i
)
236 if (buf
->page_list
[i
].buf
)
237 dma_free_coherent(&dev
->pdev
->dev
, PAGE_SIZE
,
238 buf
->page_list
[i
].buf
,
239 buf
->page_list
[i
].map
);
240 kfree(buf
->page_list
);
243 EXPORT_SYMBOL_GPL(mlx4_buf_free
);
245 static struct mlx4_db_pgdir
*mlx4_alloc_db_pgdir(struct device
*dma_device
)
247 struct mlx4_db_pgdir
*pgdir
;
249 pgdir
= kzalloc(sizeof *pgdir
, GFP_KERNEL
);
253 bitmap_fill(pgdir
->order1
, MLX4_DB_PER_PAGE
/ 2);
254 pgdir
->bits
[0] = pgdir
->order0
;
255 pgdir
->bits
[1] = pgdir
->order1
;
256 pgdir
->db_page
= dma_alloc_coherent(dma_device
, PAGE_SIZE
,
257 &pgdir
->db_dma
, GFP_KERNEL
);
258 if (!pgdir
->db_page
) {
266 static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir
*pgdir
,
267 struct mlx4_db
*db
, int order
)
272 for (o
= order
; o
<= 1; ++o
) {
273 i
= find_first_bit(pgdir
->bits
[o
], MLX4_DB_PER_PAGE
>> o
);
274 if (i
< MLX4_DB_PER_PAGE
>> o
)
281 clear_bit(i
, pgdir
->bits
[o
]);
286 set_bit(i
^ 1, pgdir
->bits
[order
]);
290 db
->db
= pgdir
->db_page
+ db
->index
;
291 db
->dma
= pgdir
->db_dma
+ db
->index
* 4;
297 int mlx4_db_alloc(struct mlx4_dev
*dev
, struct mlx4_db
*db
, int order
)
299 struct mlx4_priv
*priv
= mlx4_priv(dev
);
300 struct mlx4_db_pgdir
*pgdir
;
303 mutex_lock(&priv
->pgdir_mutex
);
305 list_for_each_entry(pgdir
, &priv
->pgdir_list
, list
)
306 if (!mlx4_alloc_db_from_pgdir(pgdir
, db
, order
))
309 pgdir
= mlx4_alloc_db_pgdir(&(dev
->pdev
->dev
));
315 list_add(&pgdir
->list
, &priv
->pgdir_list
);
317 /* This should never fail -- we just allocated an empty page: */
318 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir
, db
, order
));
321 mutex_unlock(&priv
->pgdir_mutex
);
325 EXPORT_SYMBOL_GPL(mlx4_db_alloc
);
327 void mlx4_db_free(struct mlx4_dev
*dev
, struct mlx4_db
*db
)
329 struct mlx4_priv
*priv
= mlx4_priv(dev
);
333 mutex_lock(&priv
->pgdir_mutex
);
338 if (db
->order
== 0 && test_bit(i
^ 1, db
->u
.pgdir
->order0
)) {
339 clear_bit(i
^ 1, db
->u
.pgdir
->order0
);
343 set_bit(i
, db
->u
.pgdir
->bits
[o
]);
345 if (bitmap_full(db
->u
.pgdir
->order1
, MLX4_DB_PER_PAGE
/ 2)) {
346 dma_free_coherent(&(dev
->pdev
->dev
), PAGE_SIZE
,
347 db
->u
.pgdir
->db_page
, db
->u
.pgdir
->db_dma
);
348 list_del(&db
->u
.pgdir
->list
);
352 mutex_unlock(&priv
->pgdir_mutex
);
354 EXPORT_SYMBOL_GPL(mlx4_db_free
);
356 int mlx4_alloc_hwq_res(struct mlx4_dev
*dev
, struct mlx4_hwq_resources
*wqres
,
357 int size
, int max_direct
)
361 err
= mlx4_db_alloc(dev
, &wqres
->db
, 1);
367 err
= mlx4_buf_alloc(dev
, size
, max_direct
, &wqres
->buf
);
371 err
= mlx4_mtt_init(dev
, wqres
->buf
.npages
, wqres
->buf
.page_shift
,
376 err
= mlx4_buf_write_mtt(dev
, &wqres
->mtt
, &wqres
->buf
);
383 mlx4_mtt_cleanup(dev
, &wqres
->mtt
);
385 mlx4_buf_free(dev
, size
, &wqres
->buf
);
387 mlx4_db_free(dev
, &wqres
->db
);
391 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res
);
393 void mlx4_free_hwq_res(struct mlx4_dev
*dev
, struct mlx4_hwq_resources
*wqres
,
396 mlx4_mtt_cleanup(dev
, &wqres
->mtt
);
397 mlx4_buf_free(dev
, size
, &wqres
->buf
);
398 mlx4_db_free(dev
, &wqres
->db
);
400 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res
);