2 * Copyright (c) 2003,2004,2020 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/slaballoc.h>
39 #include <sys/malloc.h>
41 #include <sys/vmmeter.h>
43 #include <sys/thread.h>
44 #include <sys/globaldata.h>
45 #include <sys/mpipe.h>
46 #include <sys/kthread.h>
48 struct mpipe_callback
{
49 STAILQ_ENTRY(mpipe_callback
) entry
;
50 void (*func
)(void *arg1
, void *arg2
);
55 static MALLOC_DEFINE(M_MPIPEARY
, "MPipe Array", "Auxiliary MPIPE structure");
57 static void mpipe_thread(void *arg
);
60 * Initialize a malloc pipeline for the specified malloc type and allocation
61 * size. Create an array to cache up to nom_count buffers and preallocate
65 mpipe_init(malloc_pipe_t mpipe
, malloc_type_t type
, int bytes
,
68 void (*construct
)(void *, void *),
69 void (*deconstruct
)(void *, void *),
77 nmax
= 0x7FFF0000; /* some very large number */
80 bzero(mpipe
, sizeof(struct malloc_pipe
));
83 mpipe
->mpflags
= mpflags
;
84 mpipe
->construct
= construct
;
85 mpipe
->deconstruct
= deconstruct
;
87 if ((mpflags
& MPF_NOZERO
) == 0)
88 mpipe
->mflags
|= M_ZERO
;
89 if (mpflags
& MPF_INT
)
90 mpipe
->mflags
|= M_USE_RESERVE
| M_USE_INTERRUPT_RESERVE
;
91 mpipe
->ary_count
= nnom
;
92 mpipe
->max_count
= nmax
;
93 mpipe
->array
= kmalloc(nnom
* sizeof(mpipe
->array
[0]), M_MPIPEARY
,
96 while (mpipe
->free_count
< nnom
) {
97 n
= mpipe
->free_count
;
98 mpipe
->array
[n
] = kmalloc(bytes
, mpipe
->type
, M_WAITOK
| mpipe
->mflags
);
100 construct(mpipe
->array
[n
], priv
);
102 ++mpipe
->total_count
;
104 STAILQ_INIT(&mpipe
->queue
);
106 lwkt_token_init(&mpipe
->token
, "mpipe token");
109 * Create a support thread for the mpipe queue
111 if (mpflags
& MPF_CALLBACK
) {
112 kthread_create(mpipe_thread
, mpipe
, &mpipe
->thread
,
113 "mpipe_%s", type
->ks_shortdesc
);
118 * Destroy a previously initialized mpipe. This routine can also safely be
119 * called on an uninitialized mpipe structure if it was zero'd or mpipe_done()
120 * was previously called on it.
123 mpipe_done(malloc_pipe_t mpipe
)
128 KKASSERT(mpipe
->free_count
== mpipe
->total_count
); /* no outstanding mem */
131 * Clean up the kthread
133 lwkt_gettoken(&mpipe
->token
);
134 mpipe
->mpflags
|= MPF_EXITING
;
135 while (mpipe
->thread
) {
136 wakeup(&mpipe
->queue
);
137 tsleep(mpipe
, 0, "mpipex", 1);
141 * Clean up the mpipe buffers
143 for (n
= mpipe
->free_count
- 1; n
>= 0; --n
) {
144 buf
= mpipe
->array
[n
];
145 mpipe
->array
[n
] = NULL
;
146 KKASSERT(buf
!= NULL
);
147 if (mpipe
->deconstruct
)
148 mpipe
->deconstruct(buf
, mpipe
->priv
);
149 kfree(buf
, mpipe
->type
);
151 mpipe
->free_count
= 0;
152 mpipe
->total_count
= 0;
154 kfree(mpipe
->array
, M_MPIPEARY
);
157 lwkt_reltoken(&mpipe
->token
);
158 lwkt_token_uninit(&mpipe
->token
);
162 * mpipe support thread for request failures when mpipe_alloc_callback()
165 * Only set MPF_QUEUEWAIT if entries are pending in the queue. If no entries
166 * are pending and a new entry is added, other code will set MPF_QUEUEWAIT
170 mpipe_thread(void *arg
)
172 malloc_pipe_t mpipe
= arg
;
173 struct mpipe_callback
*mcb
;
175 lwkt_gettoken(&mpipe
->token
);
176 while ((mpipe
->mpflags
& MPF_EXITING
) == 0) {
177 while (mpipe
->free_count
&&
178 (mcb
= STAILQ_FIRST(&mpipe
->queue
)) != NULL
) {
179 STAILQ_REMOVE(&mpipe
->queue
, mcb
, mpipe_callback
, entry
);
180 mcb
->func(mcb
->arg1
, mcb
->arg2
);
181 kfree(mcb
, M_MPIPEARY
);
183 if (STAILQ_FIRST(&mpipe
->queue
))
184 mpipe
->mpflags
|= MPF_QUEUEWAIT
;
185 tsleep(&mpipe
->queue
, 0, "wait", 0);
187 mpipe
->thread
= NULL
;
188 lwkt_reltoken(&mpipe
->token
);
194 * Allocate an entry (inline suppot routine). The allocation is guarenteed
195 * to return non-NULL up to the nominal count after which it may return NULL.
196 * Note that the implementation is defined to be allowed to block for short
199 * Use mpipe_alloc_callback() for non-blocking operation with a callback
200 * Use mpipe_alloc_nowait() for non-blocking operation without a callback
201 * Use mpipe_alloc_waitok() for blocking operation & guarenteed non-NULL
205 _mpipe_alloc_locked(malloc_pipe_t mpipe
, int mfailed
)
210 if ((n
= mpipe
->free_count
) != 0) {
212 * Use a free entry if it exists.
215 buf
= mpipe
->array
[n
];
216 mpipe
->array
[n
] = NULL
; /* sanity check, not absolutely needed */
217 mpipe
->free_count
= n
;
218 } else if (mpipe
->total_count
>= mpipe
->max_count
|| mfailed
) {
220 * Return NULL if we have hit our limit
225 * Otherwise try to malloc() non-blocking.
227 buf
= kmalloc(mpipe
->bytes
, mpipe
->type
, M_NOWAIT
| mpipe
->mflags
);
229 ++mpipe
->total_count
;
230 if (mpipe
->construct
)
231 mpipe
->construct(buf
, mpipe
->priv
);
238 * Nominal non-blocking mpipe allocation
241 mpipe_alloc_nowait(malloc_pipe_t mpipe
)
245 lwkt_gettoken(&mpipe
->token
);
246 buf
= _mpipe_alloc_locked(mpipe
, 0);
247 lwkt_reltoken(&mpipe
->token
);
253 * non-blocking mpipe allocation with callback for retry.
255 * If NULL is returned func(arg) is queued and will be called back when
256 * space is likely (but not necessarily) available.
258 * If non-NULL is returned func(arg) is ignored.
261 mpipe_alloc_callback(malloc_pipe_t mpipe
, void (*func
)(void *arg1
, void *arg2
),
262 void *arg1
, void *arg2
)
264 struct mpipe_callback
*mcb
;
267 lwkt_gettoken(&mpipe
->token
);
268 buf
= _mpipe_alloc_locked(mpipe
, 0);
270 mcb
= kmalloc(sizeof(*mcb
), M_MPIPEARY
, M_INTWAIT
);
271 buf
= _mpipe_alloc_locked(mpipe
, 0);
276 STAILQ_INSERT_TAIL(&mpipe
->queue
, mcb
, entry
);
277 mpipe
->mpflags
|= MPF_QUEUEWAIT
; /* for mpipe_thread() */
279 kfree(mcb
, M_MPIPEARY
);
282 lwkt_reltoken(&mpipe
->token
);
288 * This function can be called to nominally wait until resources are
289 * available and mpipe_alloc_nowait() is likely to return non-NULL.
291 * NOTE: mpipe_alloc_nowait() can still return NULL.
294 mpipe_wait(malloc_pipe_t mpipe
)
296 if (mpipe
->free_count
== 0) {
297 lwkt_gettoken(&mpipe
->token
);
298 while ((mpipe
->mpflags
& MPF_EXITING
) == 0) {
299 if (mpipe
->free_count
)
301 mpipe
->mpflags
|= MPF_QUEUEWAIT
;
302 tsleep(&mpipe
->queue
, 0, "wait", 0);
304 lwkt_reltoken(&mpipe
->token
);
309 * Allocate an entry, block until the allocation succeeds. This may cause
310 * us to block waiting for a prior allocation to be freed.
313 mpipe_alloc_waitok(malloc_pipe_t mpipe
)
318 lwkt_gettoken(&mpipe
->token
);
320 while ((buf
= _mpipe_alloc_locked(mpipe
, mfailed
)) == NULL
) {
322 * Block if we have hit our limit
325 tsleep(mpipe
, 0, "mpipe1", 0);
328 lwkt_reltoken(&mpipe
->token
);
334 * Free an entry, unblock any waiters. Allow NULL.
337 mpipe_free(malloc_pipe_t mpipe
, void *buf
)
344 lwkt_gettoken(&mpipe
->token
);
345 if ((n
= mpipe
->free_count
) < mpipe
->ary_count
) {
347 * Free slot available in free array (LIFO)
349 mpipe
->array
[n
] = buf
;
351 if ((mpipe
->mpflags
& (MPF_CACHEDATA
|MPF_NOZERO
)) == 0)
352 bzero(buf
, mpipe
->bytes
);
353 if (mpipe
->mpflags
& MPF_QUEUEWAIT
) {
354 mpipe
->mpflags
&= ~MPF_QUEUEWAIT
;
355 lwkt_reltoken(&mpipe
->token
);
356 wakeup(&mpipe
->queue
);
358 lwkt_reltoken(&mpipe
->token
);
361 * Wakeup anyone blocked in mpipe_alloc_*().
363 if (mpipe
->pending
) {
369 * All the free slots are full, free the buffer directly.
371 --mpipe
->total_count
;
372 KKASSERT(mpipe
->total_count
>= mpipe
->free_count
);
373 if (mpipe
->deconstruct
)
374 mpipe
->deconstruct(buf
, mpipe
->priv
);
375 lwkt_reltoken(&mpipe
->token
);
376 kfree(buf
, mpipe
->type
);