2 * Copyright (C) 2011 Citrix Ltd.
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
7 * Contributions after 2012-01-13 are licensed under the terms of the
8 * GNU GPL, version 2 or (at your option) any later version.
13 #include <sys/resource.h>
15 #include "hw/xen_backend.h"
16 #include "sysemu/blockdev.h"
17 #include "qemu/bitmap.h"
19 #include <xen/hvm/params.h>
22 #include "sysemu/xen-mapcache.h"
26 //#define MAPCACHE_DEBUG
29 # define DPRINTF(fmt, ...) do { \
30 fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
33 # define DPRINTF(fmt, ...) do { } while (0)
37 # define MCACHE_BUCKET_SHIFT 16
38 # define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
39 #elif defined(__x86_64__)
40 # define MCACHE_BUCKET_SHIFT 20
41 # define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
43 #define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
45 /* This is the size of the virtual address space reserve to QEMU that will not
47 * From empirical tests I observed that qemu use 75MB more than the
50 #define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)
52 #define mapcache_lock() ((void)0)
53 #define mapcache_unlock() ((void)0)
55 typedef struct MapCacheEntry
{
58 unsigned long *valid_mapping
;
61 struct MapCacheEntry
*next
;
64 typedef struct MapCacheRev
{
68 QTAILQ_ENTRY(MapCacheRev
) next
;
71 typedef struct MapCache
{
73 unsigned long nr_buckets
;
74 QTAILQ_HEAD(map_cache_head
, MapCacheRev
) locked_entries
;
76 /* For most cases (>99.9%), the page address is the same. */
77 MapCacheEntry
*last_entry
;
78 unsigned long max_mcache_size
;
79 unsigned int mcache_bucket_shift
;
81 phys_offset_to_gaddr_t phys_offset_to_gaddr
;
85 static MapCache
*mapcache
;
87 static inline int test_bits(int nr
, int size
, const unsigned long *addr
)
89 unsigned long res
= find_next_zero_bit(addr
, size
+ nr
, nr
);
96 void xen_map_cache_init(phys_offset_to_gaddr_t f
, void *opaque
)
99 struct rlimit rlimit_as
;
101 mapcache
= g_malloc0(sizeof (MapCache
));
103 mapcache
->phys_offset_to_gaddr
= f
;
104 mapcache
->opaque
= opaque
;
106 QTAILQ_INIT(&mapcache
->locked_entries
);
108 if (geteuid() == 0) {
109 rlimit_as
.rlim_cur
= RLIM_INFINITY
;
110 rlimit_as
.rlim_max
= RLIM_INFINITY
;
111 mapcache
->max_mcache_size
= MCACHE_MAX_SIZE
;
113 getrlimit(RLIMIT_AS
, &rlimit_as
);
114 rlimit_as
.rlim_cur
= rlimit_as
.rlim_max
;
116 if (rlimit_as
.rlim_max
!= RLIM_INFINITY
) {
117 fprintf(stderr
, "Warning: QEMU's maximum size of virtual"
118 " memory is not infinity.\n");
120 if (rlimit_as
.rlim_max
< MCACHE_MAX_SIZE
+ NON_MCACHE_MEMORY_SIZE
) {
121 mapcache
->max_mcache_size
= rlimit_as
.rlim_max
-
122 NON_MCACHE_MEMORY_SIZE
;
124 mapcache
->max_mcache_size
= MCACHE_MAX_SIZE
;
128 setrlimit(RLIMIT_AS
, &rlimit_as
);
130 mapcache
->nr_buckets
=
131 (((mapcache
->max_mcache_size
>> XC_PAGE_SHIFT
) +
132 (1UL << (MCACHE_BUCKET_SHIFT
- XC_PAGE_SHIFT
)) - 1) >>
133 (MCACHE_BUCKET_SHIFT
- XC_PAGE_SHIFT
));
135 size
= mapcache
->nr_buckets
* sizeof (MapCacheEntry
);
136 size
= (size
+ XC_PAGE_SIZE
- 1) & ~(XC_PAGE_SIZE
- 1);
137 DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__
,
138 mapcache
->nr_buckets
, size
);
139 mapcache
->entry
= g_malloc0(size
);
142 static void xen_remap_bucket(MapCacheEntry
*entry
,
144 hwaddr address_index
)
150 hwaddr nb_pfn
= size
>> XC_PAGE_SHIFT
;
152 trace_xen_remap_bucket(address_index
);
154 pfns
= g_malloc0(nb_pfn
* sizeof (xen_pfn_t
));
155 err
= g_malloc0(nb_pfn
* sizeof (int));
157 if (entry
->vaddr_base
!= NULL
) {
158 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
159 perror("unmap fails");
163 if (entry
->valid_mapping
!= NULL
) {
164 g_free(entry
->valid_mapping
);
165 entry
->valid_mapping
= NULL
;
168 for (i
= 0; i
< nb_pfn
; i
++) {
169 pfns
[i
] = (address_index
<< (MCACHE_BUCKET_SHIFT
-XC_PAGE_SHIFT
)) + i
;
172 vaddr_base
= xc_map_foreign_bulk(xen_xc
, xen_domid
, PROT_READ
|PROT_WRITE
,
174 if (vaddr_base
== NULL
) {
175 perror("xc_map_foreign_bulk");
179 entry
->vaddr_base
= vaddr_base
;
180 entry
->paddr_index
= address_index
;
182 entry
->valid_mapping
= (unsigned long *) g_malloc0(sizeof(unsigned long) *
183 BITS_TO_LONGS(size
>> XC_PAGE_SHIFT
));
185 bitmap_zero(entry
->valid_mapping
, nb_pfn
);
186 for (i
= 0; i
< nb_pfn
; i
++) {
188 bitmap_set(entry
->valid_mapping
, i
, 1);
196 uint8_t *xen_map_cache(hwaddr phys_addr
, hwaddr size
,
199 MapCacheEntry
*entry
, *pentry
= NULL
;
200 hwaddr address_index
;
201 hwaddr address_offset
;
202 hwaddr __size
= size
;
203 bool translated
= false;
206 address_index
= phys_addr
>> MCACHE_BUCKET_SHIFT
;
207 address_offset
= phys_addr
& (MCACHE_BUCKET_SIZE
- 1);
209 trace_xen_map_cache(phys_addr
);
211 if (mapcache
->last_entry
!= NULL
&&
212 mapcache
->last_entry
->paddr_index
== address_index
&&
214 trace_xen_map_cache_return(mapcache
->last_entry
->vaddr_base
+ address_offset
);
215 return mapcache
->last_entry
->vaddr_base
+ address_offset
;
218 /* size is always a multiple of MCACHE_BUCKET_SIZE */
220 __size
= size
+ address_offset
;
221 if (__size
% MCACHE_BUCKET_SIZE
) {
222 __size
+= MCACHE_BUCKET_SIZE
- (__size
% MCACHE_BUCKET_SIZE
);
225 __size
= MCACHE_BUCKET_SIZE
;
228 entry
= &mapcache
->entry
[address_index
% mapcache
->nr_buckets
];
230 while (entry
&& entry
->lock
&& entry
->vaddr_base
&&
231 (entry
->paddr_index
!= address_index
|| entry
->size
!= __size
||
232 !test_bits(address_offset
>> XC_PAGE_SHIFT
, size
>> XC_PAGE_SHIFT
,
233 entry
->valid_mapping
))) {
238 entry
= g_malloc0(sizeof (MapCacheEntry
));
239 pentry
->next
= entry
;
240 xen_remap_bucket(entry
, __size
, address_index
);
241 } else if (!entry
->lock
) {
242 if (!entry
->vaddr_base
|| entry
->paddr_index
!= address_index
||
243 entry
->size
!= __size
||
244 !test_bits(address_offset
>> XC_PAGE_SHIFT
, size
>> XC_PAGE_SHIFT
,
245 entry
->valid_mapping
)) {
246 xen_remap_bucket(entry
, __size
, address_index
);
250 if(!test_bits(address_offset
>> XC_PAGE_SHIFT
, size
>> XC_PAGE_SHIFT
,
251 entry
->valid_mapping
)) {
252 mapcache
->last_entry
= NULL
;
253 if (!translated
&& mapcache
->phys_offset_to_gaddr
) {
254 phys_addr
= mapcache
->phys_offset_to_gaddr(phys_addr
, size
, mapcache
->opaque
);
258 trace_xen_map_cache_return(NULL
);
262 mapcache
->last_entry
= entry
;
264 MapCacheRev
*reventry
= g_malloc0(sizeof(MapCacheRev
));
266 reventry
->vaddr_req
= mapcache
->last_entry
->vaddr_base
+ address_offset
;
267 reventry
->paddr_index
= mapcache
->last_entry
->paddr_index
;
268 reventry
->size
= entry
->size
;
269 QTAILQ_INSERT_HEAD(&mapcache
->locked_entries
, reventry
, next
);
272 trace_xen_map_cache_return(mapcache
->last_entry
->vaddr_base
+ address_offset
);
273 return mapcache
->last_entry
->vaddr_base
+ address_offset
;
276 ram_addr_t
xen_ram_addr_from_mapcache(void *ptr
)
278 MapCacheEntry
*entry
= NULL
;
279 MapCacheRev
*reventry
;
284 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
285 if (reventry
->vaddr_req
== ptr
) {
286 paddr_index
= reventry
->paddr_index
;
287 size
= reventry
->size
;
293 fprintf(stderr
, "%s, could not find %p\n", __func__
, ptr
);
294 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
295 DPRINTF(" "TARGET_FMT_plx
" -> %p is present\n", reventry
->paddr_index
,
296 reventry
->vaddr_req
);
302 entry
= &mapcache
->entry
[paddr_index
% mapcache
->nr_buckets
];
303 while (entry
&& (entry
->paddr_index
!= paddr_index
|| entry
->size
!= size
)) {
307 DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr
);
310 return (reventry
->paddr_index
<< MCACHE_BUCKET_SHIFT
) +
311 ((unsigned long) ptr
- (unsigned long) entry
->vaddr_base
);
314 void xen_invalidate_map_cache_entry(uint8_t *buffer
)
316 MapCacheEntry
*entry
= NULL
, *pentry
= NULL
;
317 MapCacheRev
*reventry
;
322 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
323 if (reventry
->vaddr_req
== buffer
) {
324 paddr_index
= reventry
->paddr_index
;
325 size
= reventry
->size
;
331 DPRINTF("%s, could not find %p\n", __func__
, buffer
);
332 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
333 DPRINTF(" "TARGET_FMT_plx
" -> %p is present\n", reventry
->paddr_index
, reventry
->vaddr_req
);
337 QTAILQ_REMOVE(&mapcache
->locked_entries
, reventry
, next
);
340 if (mapcache
->last_entry
!= NULL
&&
341 mapcache
->last_entry
->paddr_index
== paddr_index
) {
342 mapcache
->last_entry
= NULL
;
345 entry
= &mapcache
->entry
[paddr_index
% mapcache
->nr_buckets
];
346 while (entry
&& (entry
->paddr_index
!= paddr_index
|| entry
->size
!= size
)) {
351 DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer
);
355 if (entry
->lock
> 0 || pentry
== NULL
) {
359 pentry
->next
= entry
->next
;
360 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
361 perror("unmap fails");
364 g_free(entry
->valid_mapping
);
368 void xen_invalidate_map_cache(void)
371 MapCacheRev
*reventry
;
373 /* Flush pending AIO before destroying the mapcache */
376 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
377 DPRINTF("There should be no locked mappings at this time, "
378 "but "TARGET_FMT_plx
" -> %p is present\n",
379 reventry
->paddr_index
, reventry
->vaddr_req
);
384 for (i
= 0; i
< mapcache
->nr_buckets
; i
++) {
385 MapCacheEntry
*entry
= &mapcache
->entry
[i
];
387 if (entry
->vaddr_base
== NULL
) {
390 if (entry
->lock
> 0) {
394 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
395 perror("unmap fails");
399 entry
->paddr_index
= 0;
400 entry
->vaddr_base
= NULL
;
402 g_free(entry
->valid_mapping
);
403 entry
->valid_mapping
= NULL
;
406 mapcache
->last_entry
= NULL
;