2 * Copyright (c) 2004, 2005 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Hiten Pandya <hmp@backplane.com> and Matthew Dillon
6 * <dillon@backplane.com>.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * The MSF_BUF API was augmented from the SFBUF API:
36 * Copyright (c) 1998 David Greenman. All rights reserved.
37 * src/sys/kern/kern_sfbuf.c,v 1.7 2004/05/13 19:46:18 dillon
39 * $DragonFly: src/sys/kern/kern_msfbuf.c,v 1.16 2005/08/01 20:05:53 hmp Exp $
42 * MSFBUFs cache linear multi-page ephermal mappings and operate similar
43 * to SFBUFs. MSFBUFs use XIO's internally to hold the page list and can
44 * be considered to be a KVA wrapper around an XIO.
46 * Like the SFBUF subsystem, the locking and validation of the page array
47 * is the responsibility of the caller. Also like the SFBUF subsystem,
48 * MSFBUFs are SMP-friendly, cache the mappings, and will avoid unnecessary
49 * page invalidations when possible.
51 * MSFBUFs are primarily designed to be used in subsystems that manipulate
52 * XIOs. The DEV and BUF subsystems are a good example.
55 * - Overload XIOs representitive of smaller chunks of memory onto the
56 * same KVA space to efficiently cache smaller mappings (filesystem
57 * blocks / buffer cache related).
60 #include <sys/param.h>
61 #include <sys/systm.h>
62 #include <sys/globaldata.h>
63 #include <sys/kernel.h>
64 #include <sys/malloc.h>
65 #include <sys/queue.h>
66 #include <sys/sfbuf.h>
67 #include <sys/sysctl.h>
68 #include <sys/thread.h>
70 #include <sys/msfbuf.h>
74 #include <vm/vm_param.h>
75 #include <vm/vm_extern.h>
76 #include <vm/vm_kern.h>
77 #include <vm/vm_page.h>
80 #include <sys/thread2.h>
81 #include <vm/vm_page2.h>
83 MALLOC_DEFINE(M_MSFBUF
, "MSFBUF", "direct-copy buffers");
85 /* lists and queues associated with msf_bufs */
86 LIST_HEAD(msf_buf_list
, msf_buf
);
88 TAILQ_HEAD(, msf_buf
) msf_buf_freelist
;
90 /* indicate shortage of available msf_bufs */
91 static u_int msf_buf_alloc_want
;
93 /* base of the msf_buf map */
94 static vm_offset_t msf_base
;
95 static struct msf_buf
*msf_bufs
;
96 static int msf_buf_hits
;
97 static int msf_buf_misses
;
99 static int msf_buf_count
= 256; /* magic value */
100 SYSCTL_INT(_kern_ipc
, OID_AUTO
, msf_bufs
, CTLFLAG_RD
, &msf_buf_count
,
101 0, "number of direct-copy buffers available");
102 SYSCTL_INT(_kern_ipc
, OID_AUTO
, msf_hits
, CTLFLAG_RD
, &msf_buf_hits
,
103 0, "direct-copy buffers cache hits");
104 SYSCTL_INT(_kern_ipc
, OID_AUTO
, msf_misses
, CTLFLAG_RD
, &msf_buf_misses
,
105 0, "direct-copy buffers cache misses");
108 msf_buf_init(void *__dummy
)
113 msf_buf_alloc_want
= 0;
114 TUNABLE_INT_FETCH("kern.ipc.msfbufs", &msf_buf_count
);
116 TAILQ_INIT(&msf_buf_freelist
);
118 msf_base
= kmem_alloc_nofault(kernel_map
,
119 msf_buf_count
* XIO_INTERNAL_SIZE
);
121 msf_bufs
= malloc(msf_buf_count
* sizeof(struct msf_buf
), M_MSFBUF
,
124 /* Initialize the free list with necessary information. */
125 for (i
= 0; i
< msf_buf_count
; i
++) {
127 msf
->ms_kva
= msf_base
+ i
* XIO_INTERNAL_SIZE
;
128 msf
->ms_flags
= MSF_ONFREEQ
;
129 msf
->ms_type
= MSF_TYPE_UNKNOWN
;
130 msf
->ms_xio
= &msf
->ms_internal_xio
;
131 xio_init(&msf
->ms_internal_xio
);
132 TAILQ_INSERT_TAIL(&msf_buf_freelist
, &msf_bufs
[i
], free_list
);
135 SYSINIT(msf_buf
, SI_SUB_MBUF
, SI_ORDER_ANY
, msf_buf_init
, NULL
);
138 * Get an msf_buf from the freelist; if none are available
139 * than it will block.
141 * If SFB_CATCH was specified in 'flags' than the sleep is
142 * block is interruptable by signals etc; this flag is normally
143 * use for system calls.
146 static struct msf_buf
*
147 msf_alloc(vm_page_t firstpage
, int flags
)
154 if (firstpage
&& (msf
= firstpage
->msf_hint
) != NULL
&&
155 (msf
->ms_flags
& MSF_ONFREEQ
)
157 KKASSERT(msf
->ms_refcnt
== 0);
158 msf
->ms_flags
&= ~MSF_ONFREEQ
;
160 TAILQ_REMOVE(&msf_buf_freelist
, msf
, free_list
);
165 * Get a buffer off the freelist. If the freelist is empty, we
166 * block until something becomes available; this happens quite
167 * quickly anyway because MSFBUFs are supposed to be temporary
170 * If the SFB_CATCH flag was provided, then we allow the sleep
171 * to be interruptible.
174 if ((msf
= TAILQ_FIRST(&msf_buf_freelist
)) != NULL
) {
175 KKASSERT(msf
->ms_refcnt
== 0);
177 TAILQ_REMOVE(&msf_buf_freelist
, msf
, free_list
);
178 msf
->ms_flags
&= ~MSF_ONFREEQ
;
181 firstpage
->msf_hint
= msf
;
184 pflags
= (flags
& SFB_CATCH
) ? PCATCH
: 0;
185 ++msf_buf_alloc_want
;
186 error
= tsleep(&msf_buf_freelist
, pflags
, "msfbuf", 0);
187 --msf_buf_alloc_want
;
199 msf_map_msf(struct msf_buf
*msf
, int flags
)
202 if (flags
& SFB_CPUPRIVATE
) {
203 pmap_qenter2(msf
->ms_kva
, msf
->ms_xio
->xio_pages
,
204 msf
->ms_xio
->xio_npages
, &msf
->ms_cpumask
);
206 pmap_qenter(msf
->ms_kva
, msf
->ms_xio
->xio_pages
,
207 msf
->ms_xio
->xio_npages
);
208 msf
->ms_cpumask
= (cpumask_t
)-1;
211 pmap_qenter2(msf
->ms_kva
, msf
->ms_xio
->xio_pages
,
212 msf
->ms_xio
->xio_npages
, &msf
->ms_cpumask
);
217 msf_map_pagelist(struct msf_buf
**msfp
, vm_page_t
*list
, int npages
, int flags
)
222 KKASSERT(npages
!= 0 && npages
<= XIO_INTERNAL_PAGES
);
224 if ((msf
= msf_alloc(list
[0], flags
)) != NULL
) {
225 KKASSERT(msf
->ms_xio
== &msf
->ms_internal_xio
);
226 for (i
= 0; i
< npages
; ++i
)
227 msf
->ms_internal_xio
.xio_pages
[i
] = list
[i
];
228 msf
->ms_internal_xio
.xio_offset
= 0;
229 msf
->ms_internal_xio
.xio_npages
= npages
;
230 msf
->ms_internal_xio
.xio_bytes
= npages
<< PAGE_SHIFT
;
231 msf
->ms_type
= MSF_TYPE_PGLIST
;
232 msf_map_msf(msf
, flags
);
242 msf_map_xio(struct msf_buf
**msfp
, struct xio
*xio
, int flags
)
246 KKASSERT(xio
!= NULL
&& xio
->xio_npages
> 0);
247 KKASSERT(xio
->xio_npages
<= XIO_INTERNAL_PAGES
);
249 if ((msf
= msf_alloc(xio
->xio_pages
[0], flags
)) != NULL
) {
250 msf
->ms_type
= MSF_TYPE_XIO
;
252 msf_map_msf(msf
, flags
);
262 msf_map_ubuf(struct msf_buf
**msfp
, void *base
, size_t nbytes
, int flags
)
268 if (((int)(intptr_t)base
& PAGE_MASK
) + nbytes
> XIO_INTERNAL_SIZE
) {
273 if ((paddr
= pmap_kextract((vm_offset_t
)base
)) != 0)
274 msf
= msf_alloc(PHYS_TO_VM_PAGE(paddr
), flags
);
276 msf
= msf_alloc(NULL
, flags
);
281 error
= xio_init_ubuf(&msf
->ms_internal_xio
, base
, nbytes
, 0);
283 KKASSERT(msf
->ms_xio
== &msf
->ms_internal_xio
);
284 msf_map_msf(msf
, flags
);
285 msf
->ms_type
= MSF_TYPE_UBUF
;
296 msf_map_kbuf(struct msf_buf
**msfp
, void *base
, size_t nbytes
, int flags
)
302 if (((int)(intptr_t)base
& PAGE_MASK
) + nbytes
> XIO_INTERNAL_SIZE
) {
307 if ((paddr
= pmap_kextract((vm_offset_t
)base
)) != 0)
308 msf
= msf_alloc(PHYS_TO_VM_PAGE(paddr
), flags
);
310 msf
= msf_alloc(NULL
, flags
);
315 error
= xio_init_kbuf(&msf
->ms_internal_xio
, base
, nbytes
);
317 KKASSERT(msf
->ms_xio
== &msf
->ms_internal_xio
);
318 msf_map_msf(msf
, flags
);
319 msf
->ms_type
= MSF_TYPE_KBUF
;
330 * Iterate through the specified uio calling the function with a kernel buffer
331 * containing the data until the uio has been exhausted. If the uio
332 * represents system space no mapping occurs. If the uio represents user
333 * space the data is mapped into system space in chunks. This function does
334 * not guarentee any particular alignment or minimum chunk size, it will
335 * depend on the limitations of MSF buffers and the breakdown of the UIO's
339 msf_uio_iterate(struct uio
*uio
,
340 int (*callback
)(void *info
, char *buf
, int bytes
), void *info
)
350 switch (uio
->uio_segflg
) {
353 for (i
= 0; i
< uio
->uio_iovcnt
&& error
== 0; ++i
) {
354 iov
= &uio
->uio_iov
[i
];
356 pgoff
= (int)(intptr_t)iov
->iov_base
& PAGE_MASK
;
357 while (offset
< iov
->iov_len
) {
358 bytes
= iov
->iov_len
- offset
;
359 if (bytes
+ pgoff
> XIO_INTERNAL_SIZE
)
360 bytes
= XIO_INTERNAL_SIZE
- pgoff
;
361 error
= msf_map_ubuf(&msf
, iov
->iov_base
+ offset
, bytes
, 0);
364 error
= callback(info
, msf_buf_kva(msf
), bytes
);
375 for (i
= 0; i
< uio
->uio_iovcnt
; ++i
) {
376 iov
= &uio
->uio_iov
[i
];
377 if (iov
->iov_len
== 0)
379 error
= callback(info
, iov
->iov_base
, iov
->iov_len
);
393 * Add a reference to a buffer (currently unused)
396 msf_buf_ref(struct msf_buf
*msf
)
398 if (msf
->ms_refcnt
== 0)
399 panic("msf_buf_ref: referencing a free msf_buf");
407 * Lose a reference to an msf_buf. When none left, detach mapped page
408 * and release resources back to the system. Note that the sfbuf's
409 * removal from the freelist is delayed, so it may in fact already be
410 * on the free list. This is the optimal (and most likely) scenario.
413 msf_buf_free(struct msf_buf
*msf
)
415 KKASSERT(msf
->ms_refcnt
> 0);
418 if (--msf
->ms_refcnt
== 0) {
419 KKASSERT((msf
->ms_flags
& MSF_ONFREEQ
) == 0);
421 if (msf
->ms_type
== MSF_TYPE_UBUF
|| msf
->ms_type
== MSF_TYPE_KBUF
)
422 xio_release(msf
->ms_xio
);
424 msf
->ms_type
= MSF_TYPE_UNKNOWN
;
425 msf
->ms_flags
|= MSF_ONFREEQ
;
426 msf
->ms_xio
= &msf
->ms_internal_xio
;
427 TAILQ_INSERT_TAIL(&msf_buf_freelist
, msf
, free_list
);
429 if (msf_buf_alloc_want
> 0)
430 wakeup_one(&msf_buf_freelist
);