2 * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 static u32
convert_access(int acc
)
38 return (acc
& IB_ACCESS_REMOTE_ATOMIC
? MLX4_PERM_ATOMIC
: 0) |
39 (acc
& IB_ACCESS_REMOTE_WRITE
? MLX4_PERM_REMOTE_WRITE
: 0) |
40 (acc
& IB_ACCESS_REMOTE_READ
? MLX4_PERM_REMOTE_READ
: 0) |
41 (acc
& IB_ACCESS_LOCAL_WRITE
? MLX4_PERM_LOCAL_WRITE
: 0) |
45 struct ib_mr
*mlx4_ib_get_dma_mr(struct ib_pd
*pd
, int acc
)
47 struct mlx4_ib_mr
*mr
;
50 mr
= kmalloc(sizeof *mr
, GFP_KERNEL
);
52 return ERR_PTR(-ENOMEM
);
54 err
= mlx4_mr_alloc(to_mdev(pd
->device
)->dev
, to_mpd(pd
)->pdn
, 0,
55 ~0ull, convert_access(acc
), 0, 0, &mr
->mmr
);
59 err
= mlx4_mr_enable(to_mdev(pd
->device
)->dev
, &mr
->mmr
);
63 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= mr
->mmr
.key
;
69 mlx4_mr_free(to_mdev(pd
->device
)->dev
, &mr
->mmr
);
77 int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev
*dev
, struct mlx4_mtt
*mtt
,
81 struct ib_umem_chunk
*chunk
;
87 pages
= (u64
*) __get_free_page(GFP_KERNEL
);
93 list_for_each_entry(chunk
, &umem
->chunk_list
, list
)
94 for (j
= 0; j
< chunk
->nmap
; ++j
) {
95 len
= sg_dma_len(&chunk
->page_list
[j
]) >> mtt
->page_shift
;
96 for (k
= 0; k
< len
; ++k
) {
97 pages
[i
++] = sg_dma_address(&chunk
->page_list
[j
]) +
100 * Be friendly to mlx4_write_mtt() and
101 * pass it chunks of appropriate size.
103 if (i
== PAGE_SIZE
/ sizeof (u64
)) {
104 err
= mlx4_write_mtt(dev
->dev
, mtt
, n
,
115 err
= mlx4_write_mtt(dev
->dev
, mtt
, n
, i
, pages
);
118 free_page((unsigned long) pages
);
122 struct ib_mr
*mlx4_ib_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
123 u64 virt_addr
, int access_flags
,
124 struct ib_udata
*udata
)
126 struct mlx4_ib_dev
*dev
= to_mdev(pd
->device
);
127 struct mlx4_ib_mr
*mr
;
132 mr
= kmalloc(sizeof *mr
, GFP_KERNEL
);
134 return ERR_PTR(-ENOMEM
);
136 mr
->umem
= ib_umem_get(pd
->uobject
->context
, start
, length
,
138 if (IS_ERR(mr
->umem
)) {
139 err
= PTR_ERR(mr
->umem
);
143 n
= ib_umem_page_count(mr
->umem
);
144 shift
= ilog2(mr
->umem
->page_size
);
146 err
= mlx4_mr_alloc(dev
->dev
, to_mpd(pd
)->pdn
, virt_addr
, length
,
147 convert_access(access_flags
), n
, shift
, &mr
->mmr
);
151 err
= mlx4_ib_umem_write_mtt(dev
, &mr
->mmr
.mtt
, mr
->umem
);
155 err
= mlx4_mr_enable(dev
->dev
, &mr
->mmr
);
159 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= mr
->mmr
.key
;
164 mlx4_mr_free(to_mdev(pd
->device
)->dev
, &mr
->mmr
);
167 ib_umem_release(mr
->umem
);
175 int mlx4_ib_dereg_mr(struct ib_mr
*ibmr
)
177 struct mlx4_ib_mr
*mr
= to_mmr(ibmr
);
179 mlx4_mr_free(to_mdev(ibmr
->device
)->dev
, &mr
->mmr
);
181 ib_umem_release(mr
->umem
);
187 struct ib_mr
*mlx4_ib_alloc_fast_reg_mr(struct ib_pd
*pd
,
188 int max_page_list_len
)
190 struct mlx4_ib_dev
*dev
= to_mdev(pd
->device
);
191 struct mlx4_ib_mr
*mr
;
194 mr
= kmalloc(sizeof *mr
, GFP_KERNEL
);
196 return ERR_PTR(-ENOMEM
);
198 err
= mlx4_mr_alloc(dev
->dev
, to_mpd(pd
)->pdn
, 0, 0, 0,
199 max_page_list_len
, 0, &mr
->mmr
);
203 err
= mlx4_mr_enable(dev
->dev
, &mr
->mmr
);
207 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= mr
->mmr
.key
;
212 mlx4_mr_free(dev
->dev
, &mr
->mmr
);
219 struct ib_fast_reg_page_list
*mlx4_ib_alloc_fast_reg_page_list(struct ib_device
*ibdev
,
222 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
223 struct mlx4_ib_fast_reg_page_list
*mfrpl
;
224 int size
= page_list_len
* sizeof (u64
);
226 if (size
> PAGE_SIZE
)
227 return ERR_PTR(-EINVAL
);
229 mfrpl
= kmalloc(sizeof *mfrpl
, GFP_KERNEL
);
231 return ERR_PTR(-ENOMEM
);
233 mfrpl
->ibfrpl
.page_list
= dma_alloc_coherent(&dev
->dev
->pdev
->dev
,
236 if (!mfrpl
->ibfrpl
.page_list
)
239 WARN_ON(mfrpl
->map
& 0x3f);
241 return &mfrpl
->ibfrpl
;
245 return ERR_PTR(-ENOMEM
);
248 void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list
*page_list
)
250 struct mlx4_ib_dev
*dev
= to_mdev(page_list
->device
);
251 struct mlx4_ib_fast_reg_page_list
*mfrpl
= to_mfrpl(page_list
);
252 int size
= page_list
->max_page_list_len
* sizeof (u64
);
254 dma_free_coherent(&dev
->dev
->pdev
->dev
, size
, page_list
->page_list
,
259 struct ib_fmr
*mlx4_ib_fmr_alloc(struct ib_pd
*pd
, int acc
,
260 struct ib_fmr_attr
*fmr_attr
)
262 struct mlx4_ib_dev
*dev
= to_mdev(pd
->device
);
263 struct mlx4_ib_fmr
*fmr
;
266 fmr
= kmalloc(sizeof *fmr
, GFP_KERNEL
);
268 return ERR_PTR(-ENOMEM
);
270 err
= mlx4_fmr_alloc(dev
->dev
, to_mpd(pd
)->pdn
, convert_access(acc
),
271 fmr_attr
->max_pages
, fmr_attr
->max_maps
,
272 fmr_attr
->page_shift
, &fmr
->mfmr
);
276 err
= mlx4_fmr_enable(to_mdev(pd
->device
)->dev
, &fmr
->mfmr
);
280 fmr
->ibfmr
.rkey
= fmr
->ibfmr
.lkey
= fmr
->mfmr
.mr
.key
;
285 mlx4_mr_free(to_mdev(pd
->device
)->dev
, &fmr
->mfmr
.mr
);
293 int mlx4_ib_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
,
294 int npages
, u64 iova
)
296 struct mlx4_ib_fmr
*ifmr
= to_mfmr(ibfmr
);
297 struct mlx4_ib_dev
*dev
= to_mdev(ifmr
->ibfmr
.device
);
299 return mlx4_map_phys_fmr(dev
->dev
, &ifmr
->mfmr
, page_list
, npages
, iova
,
300 &ifmr
->ibfmr
.lkey
, &ifmr
->ibfmr
.rkey
);
303 int mlx4_ib_unmap_fmr(struct list_head
*fmr_list
)
305 struct ib_fmr
*ibfmr
;
307 struct mlx4_dev
*mdev
= NULL
;
309 list_for_each_entry(ibfmr
, fmr_list
, list
) {
310 if (mdev
&& to_mdev(ibfmr
->device
)->dev
!= mdev
)
312 mdev
= to_mdev(ibfmr
->device
)->dev
;
318 list_for_each_entry(ibfmr
, fmr_list
, list
) {
319 struct mlx4_ib_fmr
*ifmr
= to_mfmr(ibfmr
);
321 mlx4_fmr_unmap(mdev
, &ifmr
->mfmr
, &ifmr
->ibfmr
.lkey
, &ifmr
->ibfmr
.rkey
);
325 * Make sure all MPT status updates are visible before issuing
326 * SYNC_TPT firmware command.
330 err
= mlx4_SYNC_TPT(mdev
);
332 printk(KERN_WARNING
"mlx4_ib: SYNC_TPT error %d when "
333 "unmapping FMRs\n", err
);
338 int mlx4_ib_fmr_dealloc(struct ib_fmr
*ibfmr
)
340 struct mlx4_ib_fmr
*ifmr
= to_mfmr(ibfmr
);
341 struct mlx4_ib_dev
*dev
= to_mdev(ibfmr
->device
);
344 err
= mlx4_fmr_free(dev
->dev
, &ifmr
->mfmr
);