2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/highmem.h>
34 #include <linux/gfp.h>
38 struct rds_page_remainder
{
40 unsigned long r_offset
;
43 DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder
, rds_page_remainders
);
46 * returns 0 on success or -errno on failure.
48 * We don't have to worry about flush_dcache_page() as this only works
49 * with private pages. If, say, we were to do directed receive to pinned
50 * user pages we'd have to worry more about cache coherence. (Though
51 * the flush_dcache_page() in get_user_pages() would probably be enough).
53 int rds_page_copy_user(struct page
*page
, unsigned long offset
,
54 void __user
*ptr
, unsigned long bytes
,
61 rds_stats_add(s_copy_to_user
, bytes
);
63 rds_stats_add(s_copy_from_user
, bytes
);
65 addr
= kmap_atomic(page
, KM_USER0
);
67 ret
= __copy_to_user_inatomic(ptr
, addr
+ offset
, bytes
);
69 ret
= __copy_from_user_inatomic(addr
+ offset
, ptr
, bytes
);
70 kunmap_atomic(addr
, KM_USER0
);
75 ret
= copy_to_user(ptr
, addr
+ offset
, bytes
);
77 ret
= copy_from_user(addr
+ offset
, ptr
, bytes
);
85 EXPORT_SYMBOL_GPL(rds_page_copy_user
);
88 * Message allocation uses this to build up regions of a message.
90 * @bytes - the number of bytes needed.
91 * @gfp - the waiting behaviour of the allocation
93 * @gfp is always ored with __GFP_HIGHMEM. Callers must be prepared to
94 * kmap the pages, etc.
96 * If @bytes is at least a full page then this just returns a page from
99 * If @bytes is a partial page then this stores the unused region of the
100 * page in a per-cpu structure. Future partial-page allocations may be
101 * satisfied from that cached region. This lets us waste less memory on
102 * small allocations with minimal complexity. It works because the transmit
103 * path passes read-only page regions down to devices. They hold a page
104 * reference until they are done with the region.
106 int rds_page_remainder_alloc(struct scatterlist
*scat
, unsigned long bytes
,
109 struct rds_page_remainder
*rem
;
114 gfp
|= __GFP_HIGHMEM
;
116 /* jump straight to allocation if we're trying for a huge page */
117 if (bytes
>= PAGE_SIZE
) {
118 page
= alloc_page(gfp
);
122 sg_set_page(scat
, page
, PAGE_SIZE
, 0);
128 rem
= &per_cpu(rds_page_remainders
, get_cpu());
129 local_irq_save(flags
);
132 /* avoid a tiny region getting stuck by tossing it */
133 if (rem
->r_page
&& bytes
> (PAGE_SIZE
- rem
->r_offset
)) {
134 rds_stats_inc(s_page_remainder_miss
);
135 __free_page(rem
->r_page
);
139 /* hand out a fragment from the cached page */
140 if (rem
->r_page
&& bytes
<= (PAGE_SIZE
- rem
->r_offset
)) {
141 sg_set_page(scat
, rem
->r_page
, bytes
, rem
->r_offset
);
142 get_page(sg_page(scat
));
144 if (rem
->r_offset
!= 0)
145 rds_stats_inc(s_page_remainder_hit
);
147 rem
->r_offset
+= bytes
;
148 if (rem
->r_offset
== PAGE_SIZE
) {
149 __free_page(rem
->r_page
);
156 /* alloc if there is nothing for us to use */
157 local_irq_restore(flags
);
160 page
= alloc_page(gfp
);
162 rem
= &per_cpu(rds_page_remainders
, get_cpu());
163 local_irq_save(flags
);
170 /* did someone race to fill the remainder before us? */
176 /* otherwise install our page and loop around to alloc */
181 local_irq_restore(flags
);
184 rdsdebug("bytes %lu ret %d %p %u %u\n", bytes
, ret
,
185 ret
? NULL
: sg_page(scat
), ret
? 0 : scat
->offset
,
186 ret
? 0 : scat
->length
);
189 EXPORT_SYMBOL_GPL(rds_page_remainder_alloc
);
191 static int rds_page_remainder_cpu_notify(struct notifier_block
*self
,
192 unsigned long action
, void *hcpu
)
194 struct rds_page_remainder
*rem
;
195 long cpu
= (long)hcpu
;
197 rem
= &per_cpu(rds_page_remainders
, cpu
);
199 rdsdebug("cpu %ld action 0x%lx\n", cpu
, action
);
204 __free_page(rem
->r_page
);
212 static struct notifier_block rds_page_remainder_nb
= {
213 .notifier_call
= rds_page_remainder_cpu_notify
,
216 void rds_page_exit(void)
220 for_each_possible_cpu(i
)
221 rds_page_remainder_cpu_notify(&rds_page_remainder_nb
,
222 (unsigned long)CPU_DEAD
,