1 /******************************************************************************
4 * Granting foreign access to our memory reservation.
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 #include <linux/module.h>
35 #include <linux/sched.h>
37 #include <linux/slab.h>
38 #include <linux/vmalloc.h>
39 #include <linux/uaccess.h>
43 #include <xen/interface/xen.h>
45 #include <xen/grant_table.h>
46 #include <xen/interface/memory.h>
47 #include <asm/xen/hypercall.h>
49 #include <asm/pgtable.h>
50 #include <asm/sync_bitops.h>
53 /* External tools reserve first few grant table entries. */
54 #define NR_RESERVED_ENTRIES 8
55 #define GNTTAB_LIST_END 0xffffffff
56 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry))
58 static grant_ref_t
**gnttab_list
;
59 static unsigned int nr_grant_frames
;
60 static unsigned int boot_max_nr_grant_frames
;
61 static int gnttab_free_count
;
62 static grant_ref_t gnttab_free_head
;
63 static DEFINE_SPINLOCK(gnttab_list_lock
);
64 unsigned long xen_hvm_resume_frames
;
65 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames
);
67 static struct grant_entry
*shared
;
69 static struct gnttab_free_callback
*gnttab_free_callback_list
;
71 static int gnttab_expand(unsigned int req_entries
);
73 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
75 static inline grant_ref_t
*__gnttab_entry(grant_ref_t entry
)
77 return &gnttab_list
[(entry
) / RPP
][(entry
) % RPP
];
79 /* This can be used as an l-value */
80 #define gnttab_entry(entry) (*__gnttab_entry(entry))
82 static int get_free_entries(unsigned count
)
88 spin_lock_irqsave(&gnttab_list_lock
, flags
);
90 if ((gnttab_free_count
< count
) &&
91 ((rc
= gnttab_expand(count
- gnttab_free_count
)) < 0)) {
92 spin_unlock_irqrestore(&gnttab_list_lock
, flags
);
96 ref
= head
= gnttab_free_head
;
97 gnttab_free_count
-= count
;
99 head
= gnttab_entry(head
);
100 gnttab_free_head
= gnttab_entry(head
);
101 gnttab_entry(head
) = GNTTAB_LIST_END
;
103 spin_unlock_irqrestore(&gnttab_list_lock
, flags
);
108 static void do_free_callbacks(void)
110 struct gnttab_free_callback
*callback
, *next
;
112 callback
= gnttab_free_callback_list
;
113 gnttab_free_callback_list
= NULL
;
115 while (callback
!= NULL
) {
116 next
= callback
->next
;
117 if (gnttab_free_count
>= callback
->count
) {
118 callback
->next
= NULL
;
119 callback
->fn(callback
->arg
);
121 callback
->next
= gnttab_free_callback_list
;
122 gnttab_free_callback_list
= callback
;
128 static inline void check_free_callbacks(void)
130 if (unlikely(gnttab_free_callback_list
))
134 static void put_free_entry(grant_ref_t ref
)
137 spin_lock_irqsave(&gnttab_list_lock
, flags
);
138 gnttab_entry(ref
) = gnttab_free_head
;
139 gnttab_free_head
= ref
;
141 check_free_callbacks();
142 spin_unlock_irqrestore(&gnttab_list_lock
, flags
);
145 static void update_grant_entry(grant_ref_t ref
, domid_t domid
,
146 unsigned long frame
, unsigned flags
)
149 * Introducing a valid entry into the grant table:
150 * 1. Write ent->domid.
151 * 2. Write ent->frame:
152 * GTF_permit_access: Frame to which access is permitted.
153 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
154 * frame, or zero if none.
155 * 3. Write memory barrier (WMB).
156 * 4. Write ent->flags, inc. valid type.
158 shared
[ref
].frame
= frame
;
159 shared
[ref
].domid
= domid
;
161 shared
[ref
].flags
= flags
;
165 * Public grant-issuing interface functions
167 void gnttab_grant_foreign_access_ref(grant_ref_t ref
, domid_t domid
,
168 unsigned long frame
, int readonly
)
170 update_grant_entry(ref
, domid
, frame
,
171 GTF_permit_access
| (readonly
? GTF_readonly
: 0));
173 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref
);
175 int gnttab_grant_foreign_access(domid_t domid
, unsigned long frame
,
180 ref
= get_free_entries(1);
181 if (unlikely(ref
< 0))
184 gnttab_grant_foreign_access_ref(ref
, domid
, frame
, readonly
);
188 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access
);
190 int gnttab_query_foreign_access(grant_ref_t ref
)
194 nflags
= shared
[ref
].flags
;
196 return (nflags
& (GTF_reading
|GTF_writing
));
198 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access
);
200 int gnttab_end_foreign_access_ref(grant_ref_t ref
, int readonly
)
204 nflags
= shared
[ref
].flags
;
207 if (flags
& (GTF_reading
|GTF_writing
)) {
208 printk(KERN_ALERT
"WARNING: g.e. still in use!\n");
211 } while ((nflags
= sync_cmpxchg(&shared
[ref
].flags
, flags
, 0)) != flags
);
215 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref
);
217 void gnttab_end_foreign_access(grant_ref_t ref
, int readonly
,
220 if (gnttab_end_foreign_access_ref(ref
, readonly
)) {
226 "WARNING: leaking g.e. and page still in use!\n");
229 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access
);
231 int gnttab_grant_foreign_transfer(domid_t domid
, unsigned long pfn
)
235 ref
= get_free_entries(1);
236 if (unlikely(ref
< 0))
238 gnttab_grant_foreign_transfer_ref(ref
, domid
, pfn
);
242 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer
);
244 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref
, domid_t domid
,
247 update_grant_entry(ref
, domid
, pfn
, GTF_accept_transfer
);
249 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref
);
251 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref
)
257 * If a transfer is not even yet started, try to reclaim the grant
258 * reference and return failure (== 0).
260 while (!((flags
= shared
[ref
].flags
) & GTF_transfer_committed
)) {
261 if (sync_cmpxchg(&shared
[ref
].flags
, flags
, 0) == flags
)
266 /* If a transfer is in progress then wait until it is completed. */
267 while (!(flags
& GTF_transfer_completed
)) {
268 flags
= shared
[ref
].flags
;
272 rmb(); /* Read the frame number /after/ reading completion status. */
273 frame
= shared
[ref
].frame
;
278 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref
);
280 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref
)
282 unsigned long frame
= gnttab_end_foreign_transfer_ref(ref
);
286 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer
);
288 void gnttab_free_grant_reference(grant_ref_t ref
)
292 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference
);
294 void gnttab_free_grant_references(grant_ref_t head
)
299 if (head
== GNTTAB_LIST_END
)
301 spin_lock_irqsave(&gnttab_list_lock
, flags
);
303 while (gnttab_entry(ref
) != GNTTAB_LIST_END
) {
304 ref
= gnttab_entry(ref
);
307 gnttab_entry(ref
) = gnttab_free_head
;
308 gnttab_free_head
= head
;
309 gnttab_free_count
+= count
;
310 check_free_callbacks();
311 spin_unlock_irqrestore(&gnttab_list_lock
, flags
);
313 EXPORT_SYMBOL_GPL(gnttab_free_grant_references
);
315 int gnttab_alloc_grant_references(u16 count
, grant_ref_t
*head
)
317 int h
= get_free_entries(count
);
326 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references
);
328 int gnttab_empty_grant_references(const grant_ref_t
*private_head
)
330 return (*private_head
== GNTTAB_LIST_END
);
332 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references
);
334 int gnttab_claim_grant_reference(grant_ref_t
*private_head
)
336 grant_ref_t g
= *private_head
;
337 if (unlikely(g
== GNTTAB_LIST_END
))
339 *private_head
= gnttab_entry(g
);
342 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference
);
344 void gnttab_release_grant_reference(grant_ref_t
*private_head
,
347 gnttab_entry(release
) = *private_head
;
348 *private_head
= release
;
350 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference
);
352 void gnttab_request_free_callback(struct gnttab_free_callback
*callback
,
353 void (*fn
)(void *), void *arg
, u16 count
)
356 spin_lock_irqsave(&gnttab_list_lock
, flags
);
361 callback
->count
= count
;
362 callback
->next
= gnttab_free_callback_list
;
363 gnttab_free_callback_list
= callback
;
364 check_free_callbacks();
366 spin_unlock_irqrestore(&gnttab_list_lock
, flags
);
368 EXPORT_SYMBOL_GPL(gnttab_request_free_callback
);
370 void gnttab_cancel_free_callback(struct gnttab_free_callback
*callback
)
372 struct gnttab_free_callback
**pcb
;
375 spin_lock_irqsave(&gnttab_list_lock
, flags
);
376 for (pcb
= &gnttab_free_callback_list
; *pcb
; pcb
= &(*pcb
)->next
) {
377 if (*pcb
== callback
) {
378 *pcb
= callback
->next
;
382 spin_unlock_irqrestore(&gnttab_list_lock
, flags
);
384 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback
);
386 static int grow_gnttab_list(unsigned int more_frames
)
388 unsigned int new_nr_grant_frames
, extra_entries
, i
;
389 unsigned int nr_glist_frames
, new_nr_glist_frames
;
391 new_nr_grant_frames
= nr_grant_frames
+ more_frames
;
392 extra_entries
= more_frames
* GREFS_PER_GRANT_FRAME
;
394 nr_glist_frames
= (nr_grant_frames
* GREFS_PER_GRANT_FRAME
+ RPP
- 1) / RPP
;
395 new_nr_glist_frames
=
396 (new_nr_grant_frames
* GREFS_PER_GRANT_FRAME
+ RPP
- 1) / RPP
;
397 for (i
= nr_glist_frames
; i
< new_nr_glist_frames
; i
++) {
398 gnttab_list
[i
] = (grant_ref_t
*)__get_free_page(GFP_ATOMIC
);
404 for (i
= GREFS_PER_GRANT_FRAME
* nr_grant_frames
;
405 i
< GREFS_PER_GRANT_FRAME
* new_nr_grant_frames
- 1; i
++)
406 gnttab_entry(i
) = i
+ 1;
408 gnttab_entry(i
) = gnttab_free_head
;
409 gnttab_free_head
= GREFS_PER_GRANT_FRAME
* nr_grant_frames
;
410 gnttab_free_count
+= extra_entries
;
412 nr_grant_frames
= new_nr_grant_frames
;
414 check_free_callbacks();
419 for ( ; i
>= nr_glist_frames
; i
--)
420 free_page((unsigned long) gnttab_list
[i
]);
424 static unsigned int __max_nr_grant_frames(void)
426 struct gnttab_query_size query
;
429 query
.dom
= DOMID_SELF
;
431 rc
= HYPERVISOR_grant_table_op(GNTTABOP_query_size
, &query
, 1);
432 if ((rc
< 0) || (query
.status
!= GNTST_okay
))
433 return 4; /* Legacy max supported number of frames */
435 return query
.max_nr_frames
;
438 unsigned int gnttab_max_grant_frames(void)
440 unsigned int xen_max
= __max_nr_grant_frames();
442 if (xen_max
> boot_max_nr_grant_frames
)
443 return boot_max_nr_grant_frames
;
446 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames
);
448 static int gnttab_map(unsigned int start_idx
, unsigned int end_idx
)
450 struct gnttab_setup_table setup
;
451 unsigned long *frames
;
452 unsigned int nr_gframes
= end_idx
+ 1;
455 if (xen_hvm_domain()) {
456 struct xen_add_to_physmap xatp
;
457 unsigned int i
= end_idx
;
460 * Loop backwards, so that the first hypercall has the largest
461 * index, ensuring that the table will grow only once.
464 xatp
.domid
= DOMID_SELF
;
466 xatp
.space
= XENMAPSPACE_grant_table
;
467 xatp
.gpfn
= (xen_hvm_resume_frames
>> PAGE_SHIFT
) + i
;
468 rc
= HYPERVISOR_memory_op(XENMEM_add_to_physmap
, &xatp
);
471 "grant table add_to_physmap failed, err=%d\n", rc
);
474 } while (i
-- > start_idx
);
479 frames
= kmalloc(nr_gframes
* sizeof(unsigned long), GFP_ATOMIC
);
483 setup
.dom
= DOMID_SELF
;
484 setup
.nr_frames
= nr_gframes
;
485 set_xen_guest_handle(setup
.frame_list
, frames
);
487 rc
= HYPERVISOR_grant_table_op(GNTTABOP_setup_table
, &setup
, 1);
493 BUG_ON(rc
|| setup
.status
);
495 rc
= arch_gnttab_map_shared(frames
, nr_gframes
, gnttab_max_grant_frames(),
504 int gnttab_resume(void)
506 unsigned int max_nr_gframes
;
508 max_nr_gframes
= gnttab_max_grant_frames();
509 if (max_nr_gframes
< nr_grant_frames
)
513 return gnttab_map(0, nr_grant_frames
- 1);
516 shared
= ioremap(xen_hvm_resume_frames
, PAGE_SIZE
* max_nr_gframes
);
517 if (shared
== NULL
) {
519 "Failed to ioremap gnttab share frames!");
524 gnttab_map(0, nr_grant_frames
- 1);
529 int gnttab_suspend(void)
531 arch_gnttab_unmap_shared(shared
, nr_grant_frames
);
535 static int gnttab_expand(unsigned int req_entries
)
538 unsigned int cur
, extra
;
540 cur
= nr_grant_frames
;
541 extra
= ((req_entries
+ (GREFS_PER_GRANT_FRAME
-1)) /
542 GREFS_PER_GRANT_FRAME
);
543 if (cur
+ extra
> gnttab_max_grant_frames())
546 rc
= gnttab_map(cur
, cur
+ extra
- 1);
548 rc
= grow_gnttab_list(extra
);
553 int gnttab_init(void)
556 unsigned int max_nr_glist_frames
, nr_glist_frames
;
557 unsigned int nr_init_grefs
;
560 boot_max_nr_grant_frames
= __max_nr_grant_frames();
562 /* Determine the maximum number of frames required for the
563 * grant reference free list on the current hypervisor.
565 max_nr_glist_frames
= (boot_max_nr_grant_frames
*
566 GREFS_PER_GRANT_FRAME
/ RPP
);
568 gnttab_list
= kmalloc(max_nr_glist_frames
* sizeof(grant_ref_t
*),
570 if (gnttab_list
== NULL
)
573 nr_glist_frames
= (nr_grant_frames
* GREFS_PER_GRANT_FRAME
+ RPP
- 1) / RPP
;
574 for (i
= 0; i
< nr_glist_frames
; i
++) {
575 gnttab_list
[i
] = (grant_ref_t
*)__get_free_page(GFP_KERNEL
);
576 if (gnttab_list
[i
] == NULL
)
580 if (gnttab_resume() < 0)
583 nr_init_grefs
= nr_grant_frames
* GREFS_PER_GRANT_FRAME
;
585 for (i
= NR_RESERVED_ENTRIES
; i
< nr_init_grefs
- 1; i
++)
586 gnttab_entry(i
) = i
+ 1;
588 gnttab_entry(nr_init_grefs
- 1) = GNTTAB_LIST_END
;
589 gnttab_free_count
= nr_init_grefs
- NR_RESERVED_ENTRIES
;
590 gnttab_free_head
= NR_RESERVED_ENTRIES
;
592 printk("Grant table initialized\n");
596 for (i
--; i
>= 0; i
--)
597 free_page((unsigned long)gnttab_list
[i
]);
601 EXPORT_SYMBOL_GPL(gnttab_init
);
603 static int __devinit
__gnttab_init(void)
605 /* Delay grant-table initialization in the PV on HVM case */
606 if (xen_hvm_domain())
609 if (!xen_pv_domain())
612 return gnttab_init();
615 core_initcall(__gnttab_init
);