2 * Linux UFFD-WP support
4 * Copyright Virtuozzo GmbH, 2020
7 * Andrey Gruzdev <andrey.gruzdev@virtuozzo.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 or
10 * later. See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
14 #include "qemu/bitops.h"
15 #include "qemu/error-report.h"
16 #include "qemu/userfaultfd.h"
19 #include <sys/syscall.h>
20 #include <sys/ioctl.h>
23 * uffd_query_features: query UFFD features
25 * Returns: 0 on success, negative value in case of an error
27 * @features: parameter to receive 'uffdio_api.features'
29 int uffd_query_features(uint64_t *features
)
32 struct uffdio_api api_struct
= { 0 };
35 uffd_fd
= syscall(__NR_userfaultfd
, O_CLOEXEC
);
37 trace_uffd_query_features_nosys(errno
);
41 api_struct
.api
= UFFD_API
;
42 api_struct
.features
= 0;
44 if (ioctl(uffd_fd
, UFFDIO_API
, &api_struct
)) {
45 trace_uffd_query_features_api_failed(errno
);
48 *features
= api_struct
.features
;
57 * uffd_create_fd: create UFFD file descriptor
59 * Returns non-negative file descriptor or negative value in case of an error
61 * @features: UFFD features to request
62 * @non_blocking: create UFFD file descriptor for non-blocking operation
64 int uffd_create_fd(uint64_t features
, bool non_blocking
)
68 struct uffdio_api api_struct
= { 0 };
69 uint64_t ioctl_mask
= BIT(_UFFDIO_REGISTER
) | BIT(_UFFDIO_UNREGISTER
);
71 flags
= O_CLOEXEC
| (non_blocking
? O_NONBLOCK
: 0);
72 uffd_fd
= syscall(__NR_userfaultfd
, flags
);
74 trace_uffd_create_fd_nosys(errno
);
78 api_struct
.api
= UFFD_API
;
79 api_struct
.features
= features
;
80 if (ioctl(uffd_fd
, UFFDIO_API
, &api_struct
)) {
81 trace_uffd_create_fd_api_failed(errno
);
84 if ((api_struct
.ioctls
& ioctl_mask
) != ioctl_mask
) {
85 trace_uffd_create_fd_api_noioctl(ioctl_mask
, api_struct
.ioctls
);
97 * uffd_close_fd: close UFFD file descriptor
99 * @uffd_fd: UFFD file descriptor
101 void uffd_close_fd(int uffd_fd
)
103 assert(uffd_fd
>= 0);
108 * uffd_register_memory: register memory range via UFFD-IO
110 * Returns 0 in case of success, negative value in case of an error
112 * @uffd_fd: UFFD file descriptor
113 * @addr: base address of memory range
114 * @length: length of memory range
115 * @mode: UFFD register mode (UFFDIO_REGISTER_MODE_MISSING, ...)
116 * @ioctls: optional pointer to receive supported IOCTL mask
118 int uffd_register_memory(int uffd_fd
, void *addr
, uint64_t length
,
119 uint64_t mode
, uint64_t *ioctls
)
121 struct uffdio_register uffd_register
;
123 uffd_register
.range
.start
= (uintptr_t) addr
;
124 uffd_register
.range
.len
= length
;
125 uffd_register
.mode
= mode
;
127 if (ioctl(uffd_fd
, UFFDIO_REGISTER
, &uffd_register
)) {
128 trace_uffd_register_memory_failed(addr
, length
, mode
, errno
);
132 *ioctls
= uffd_register
.ioctls
;
139 * uffd_unregister_memory: un-register memory range with UFFD-IO
141 * Returns 0 in case of success, negative value in case of an error
143 * @uffd_fd: UFFD file descriptor
144 * @addr: base address of memory range
145 * @length: length of memory range
147 int uffd_unregister_memory(int uffd_fd
, void *addr
, uint64_t length
)
149 struct uffdio_range uffd_range
;
151 uffd_range
.start
= (uintptr_t) addr
;
152 uffd_range
.len
= length
;
154 if (ioctl(uffd_fd
, UFFDIO_UNREGISTER
, &uffd_range
)) {
155 trace_uffd_unregister_memory_failed(addr
, length
, errno
);
163 * uffd_change_protection: protect/un-protect memory range for writes via UFFD-IO
165 * Returns 0 on success, negative value in case of error
167 * @uffd_fd: UFFD file descriptor
168 * @addr: base address of memory range
169 * @length: length of memory range
170 * @wp: write-protect/unprotect
171 * @dont_wake: do not wake threads waiting on wr-protected page
173 int uffd_change_protection(int uffd_fd
, void *addr
, uint64_t length
,
174 bool wp
, bool dont_wake
)
176 struct uffdio_writeprotect uffd_writeprotect
;
178 uffd_writeprotect
.range
.start
= (uintptr_t) addr
;
179 uffd_writeprotect
.range
.len
= length
;
180 if (!wp
&& dont_wake
) {
181 /* DONTWAKE is meaningful only on protection release */
182 uffd_writeprotect
.mode
= UFFDIO_WRITEPROTECT_MODE_DONTWAKE
;
184 uffd_writeprotect
.mode
= (wp
? UFFDIO_WRITEPROTECT_MODE_WP
: 0);
187 if (ioctl(uffd_fd
, UFFDIO_WRITEPROTECT
, &uffd_writeprotect
)) {
188 error_report("uffd_change_protection() failed: addr=%p len=%" PRIu64
189 " mode=%" PRIx64
" errno=%i", addr
, length
,
190 (uint64_t) uffd_writeprotect
.mode
, errno
);
198 * uffd_copy_page: copy range of pages to destination via UFFD-IO
200 * Copy range of source pages to the destination to resolve
201 * missing page fault somewhere in the destination range.
203 * Returns 0 on success, negative value in case of an error
205 * @uffd_fd: UFFD file descriptor
206 * @dst_addr: destination base address
207 * @src_addr: source base address
208 * @length: length of the range to copy
209 * @dont_wake: do not wake threads waiting on missing page
211 int uffd_copy_page(int uffd_fd
, void *dst_addr
, void *src_addr
,
212 uint64_t length
, bool dont_wake
)
214 struct uffdio_copy uffd_copy
;
216 uffd_copy
.dst
= (uintptr_t) dst_addr
;
217 uffd_copy
.src
= (uintptr_t) src_addr
;
218 uffd_copy
.len
= length
;
219 uffd_copy
.mode
= dont_wake
? UFFDIO_COPY_MODE_DONTWAKE
: 0;
221 if (ioctl(uffd_fd
, UFFDIO_COPY
, &uffd_copy
)) {
222 error_report("uffd_copy_page() failed: dst_addr=%p src_addr=%p length=%" PRIu64
223 " mode=%" PRIx64
" errno=%i", dst_addr
, src_addr
,
224 length
, (uint64_t) uffd_copy
.mode
, errno
);
232 * uffd_zero_page: fill range of pages with zeroes via UFFD-IO
234 * Fill range pages with zeroes to resolve missing page fault within the range.
236 * Returns 0 on success, negative value in case of an error
238 * @uffd_fd: UFFD file descriptor
239 * @addr: base address
240 * @length: length of the range to fill with zeroes
241 * @dont_wake: do not wake threads waiting on missing page
243 int uffd_zero_page(int uffd_fd
, void *addr
, uint64_t length
, bool dont_wake
)
245 struct uffdio_zeropage uffd_zeropage
;
247 uffd_zeropage
.range
.start
= (uintptr_t) addr
;
248 uffd_zeropage
.range
.len
= length
;
249 uffd_zeropage
.mode
= dont_wake
? UFFDIO_ZEROPAGE_MODE_DONTWAKE
: 0;
251 if (ioctl(uffd_fd
, UFFDIO_ZEROPAGE
, &uffd_zeropage
)) {
252 error_report("uffd_zero_page() failed: addr=%p length=%" PRIu64
253 " mode=%" PRIx64
" errno=%i", addr
, length
,
254 (uint64_t) uffd_zeropage
.mode
, errno
);
262 * uffd_wakeup: wake up threads waiting on page UFFD-managed page fault resolution
264 * Wake up threads waiting on any page/pages from the designated range.
265 * The main use case is when during some period, page faults are resolved
266 * via UFFD-IO IOCTLs with MODE_DONTWAKE flag set, then after that all waits
267 * for the whole memory range are satisfied in a single call to uffd_wakeup().
269 * Returns 0 on success, negative value in case of an error
271 * @uffd_fd: UFFD file descriptor
272 * @addr: base address
273 * @length: length of the range
275 int uffd_wakeup(int uffd_fd
, void *addr
, uint64_t length
)
277 struct uffdio_range uffd_range
;
279 uffd_range
.start
= (uintptr_t) addr
;
280 uffd_range
.len
= length
;
282 if (ioctl(uffd_fd
, UFFDIO_WAKE
, &uffd_range
)) {
283 error_report("uffd_wakeup() failed: addr=%p length=%" PRIu64
" errno=%i",
284 addr
, length
, errno
);
292 * uffd_read_events: read pending UFFD events
294 * Returns number of fetched messages, 0 if non is available or
295 * negative value in case of an error
297 * @uffd_fd: UFFD file descriptor
298 * @msgs: pointer to message buffer
299 * @count: number of messages that can fit in the buffer
301 int uffd_read_events(int uffd_fd
, struct uffd_msg
*msgs
, int count
)
305 res
= read(uffd_fd
, msgs
, count
* sizeof(struct uffd_msg
));
306 } while (res
< 0 && errno
== EINTR
);
308 if ((res
< 0 && errno
== EAGAIN
)) {
312 error_report("uffd_read_events() failed: errno=%i", errno
);
316 return (int) (res
/ sizeof(struct uffd_msg
));
320 * uffd_poll_events: poll UFFD file descriptor for read
322 * Returns true if events are available for read, false otherwise
324 * @uffd_fd: UFFD file descriptor
325 * @tmo: timeout value
327 bool uffd_poll_events(int uffd_fd
, int tmo
)
330 struct pollfd poll_fd
= { .fd
= uffd_fd
, .events
= POLLIN
, .revents
= 0 };
333 res
= poll(&poll_fd
, 1, tmo
);
334 } while (res
< 0 && errno
== EINTR
);
340 error_report("uffd_poll_events() failed: errno=%i", errno
);
344 return (poll_fd
.revents
& POLLIN
) != 0;