2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/highmem.h>
34 #include <linux/gfp.h>
38 struct rds_page_remainder
{
40 unsigned long r_offset
;
43 DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder
, rds_page_remainders
);
46 * returns 0 on success or -errno on failure.
48 * We don't have to worry about flush_dcache_page() as this only works
49 * with private pages. If, say, we were to do directed receive to pinned
50 * user pages we'd have to worry more about cache coherence. (Though
51 * the flush_dcache_page() in get_user_pages() would probably be enough).
53 int rds_page_copy_user(struct page
*page
, unsigned long offset
,
54 void __user
*ptr
, unsigned long bytes
,
62 rds_stats_add(s_copy_to_user
, bytes
);
63 ret
= copy_to_user(ptr
, addr
+ offset
, bytes
);
65 rds_stats_add(s_copy_from_user
, bytes
);
66 ret
= copy_from_user(addr
+ offset
, ptr
, bytes
);
70 return ret
? -EFAULT
: 0;
72 EXPORT_SYMBOL_GPL(rds_page_copy_user
);
75 * Message allocation uses this to build up regions of a message.
77 * @bytes - the number of bytes needed.
78 * @gfp - the waiting behaviour of the allocation
80 * @gfp is always ored with __GFP_HIGHMEM. Callers must be prepared to
81 * kmap the pages, etc.
83 * If @bytes is at least a full page then this just returns a page from
86 * If @bytes is a partial page then this stores the unused region of the
87 * page in a per-cpu structure. Future partial-page allocations may be
88 * satisfied from that cached region. This lets us waste less memory on
89 * small allocations with minimal complexity. It works because the transmit
90 * path passes read-only page regions down to devices. They hold a page
91 * reference until they are done with the region.
93 int rds_page_remainder_alloc(struct scatterlist
*scat
, unsigned long bytes
,
96 struct rds_page_remainder
*rem
;
101 gfp
|= __GFP_HIGHMEM
;
103 /* jump straight to allocation if we're trying for a huge page */
104 if (bytes
>= PAGE_SIZE
) {
105 page
= alloc_page(gfp
);
109 sg_set_page(scat
, page
, PAGE_SIZE
, 0);
115 rem
= &per_cpu(rds_page_remainders
, get_cpu());
116 local_irq_save(flags
);
119 /* avoid a tiny region getting stuck by tossing it */
120 if (rem
->r_page
&& bytes
> (PAGE_SIZE
- rem
->r_offset
)) {
121 rds_stats_inc(s_page_remainder_miss
);
122 __free_page(rem
->r_page
);
126 /* hand out a fragment from the cached page */
127 if (rem
->r_page
&& bytes
<= (PAGE_SIZE
- rem
->r_offset
)) {
128 sg_set_page(scat
, rem
->r_page
, bytes
, rem
->r_offset
);
129 get_page(sg_page(scat
));
131 if (rem
->r_offset
!= 0)
132 rds_stats_inc(s_page_remainder_hit
);
134 rem
->r_offset
+= bytes
;
135 if (rem
->r_offset
== PAGE_SIZE
) {
136 __free_page(rem
->r_page
);
143 /* alloc if there is nothing for us to use */
144 local_irq_restore(flags
);
147 page
= alloc_page(gfp
);
149 rem
= &per_cpu(rds_page_remainders
, get_cpu());
150 local_irq_save(flags
);
157 /* did someone race to fill the remainder before us? */
163 /* otherwise install our page and loop around to alloc */
168 local_irq_restore(flags
);
171 rdsdebug("bytes %lu ret %d %p %u %u\n", bytes
, ret
,
172 ret
? NULL
: sg_page(scat
), ret
? 0 : scat
->offset
,
173 ret
? 0 : scat
->length
);
177 static int rds_page_remainder_cpu_notify(struct notifier_block
*self
,
178 unsigned long action
, void *hcpu
)
180 struct rds_page_remainder
*rem
;
181 long cpu
= (long)hcpu
;
183 rem
= &per_cpu(rds_page_remainders
, cpu
);
185 rdsdebug("cpu %ld action 0x%lx\n", cpu
, action
);
190 __free_page(rem
->r_page
);
198 static struct notifier_block rds_page_remainder_nb
= {
199 .notifier_call
= rds_page_remainder_cpu_notify
,
202 void rds_page_exit(void)
206 for_each_possible_cpu(i
)
207 rds_page_remainder_cpu_notify(&rds_page_remainder_nb
,
208 (unsigned long)CPU_DEAD
,