sys/kern/subr_mbpool.c

   1 /*-
   2  * Copyright (c) 2003
   3  *      Fraunhofer Institute for Open Communication Systems (FhG Fokus).
   4  *      All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  * Author: Hartmut Brandt <harti@freebsd.org>
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/lock.h>
  35 #include <sys/mutex.h>
  36 #include <sys/kernel.h>
  37 #include <sys/systm.h>
  38 #include <sys/malloc.h>
  39 #include <sys/module.h>
  40
  41 #include <machine/bus.h>
  42
  43 #include <sys/mbpool.h>
  44
  45 MODULE_VERSION(libmbpool, 1);
  46
  47 /*
  48  * Memory is allocated as DMA-able pages. Each page is divided into a number
  49  * of equal chunks where the last 4 bytes of each chunk are occupied by
  50  * the page number and the chunk number. The caller must take these four
  51  * bytes into account when specifying the chunk size. Each page is mapped by
  52  * its own DMA map using the user specified DMA tag.
  53  *
  54  * Each chunk has a used and a card bit in the high bits of its page number.
  55  *  0    0      chunk is free and may be allocated
  56  *  1    1      chunk has been given to the interface
  57  *  0    1      chunk is traveling through the system
  58  *  1    0      illegal
  59  */
  60 struct mbtrail {
  61         uint16_t        chunk;
  62         uint16_t        page;
  63 };
  64 #define MBP_CARD        0x8000
  65 #define MBP_USED        0x4000
  66 #define MBP_PMSK        0x3fff          /* page number mask */
  67 #define MBP_CMSK        0x01ff          /* chunk number mask */
  68
  69 struct mbfree {
  70         SLIST_ENTRY(mbfree) link;       /* link on free list */
  71 };
  72
  73 struct mbpage {
  74         bus_dmamap_t    map;            /* map for this page */
  75         bus_addr_t      phy;            /* physical address */
  76         void            *va;            /* the memory */
  77 };
  78
  79 struct mbpool {
  80         const char      *name;          /* a name for this pool */
  81         bus_dma_tag_t   dmat;           /* tag for mapping */
  82         u_int           max_pages;      /* maximum number of pages */
  83         size_t          page_size;      /* size of each allocation */
  84         size_t          chunk_size;     /* size of each external mbuf */
  85
  86         struct mtx      free_lock;      /* lock of free list */
  87         SLIST_HEAD(, mbfree) free_list; /* free list */
  88         u_int           npages;         /* current number of pages */
  89         u_int           nchunks;        /* chunks per page */
  90         struct mbpage   pages[];        /* pages */
  91 };
  92
  93 static MALLOC_DEFINE(M_MBPOOL, "mbpools", "mbuf pools");
  94
  95 /*
  96  * Make a trail pointer from a chunk pointer
  97  */
  98 #define C2T(P, C)       ((struct mbtrail *)((char *)(C) + (P)->chunk_size - \
  99                             sizeof(struct mbtrail)))
 100
 101 /*
 102  * Make a free chunk pointer from a chunk number
 103  */
 104 #define N2C(P, PG, C)   ((struct mbfree *)((char *)(PG)->va + \
 105                             (C) * (P)->chunk_size))
 106
 107 /*
 108  * Make/parse handles
 109  */
 110 #define HMAKE(P, C)     ((((P) & MBP_PMSK) << 16) | ((C) << 7))
 111 #define HPAGE(H)        (((H) >> 16) & MBP_PMSK)
 112 #define HCHUNK(H)       (((H) >>  7) & MBP_CMSK)
 113
 114 /*
 115  * initialize a pool
 116  */
 117 int
 118 mbp_create(struct mbpool **pp, const char *name, bus_dma_tag_t dmat,
 119     u_int max_pages, size_t page_size, size_t chunk_size)
 120 {
 121         u_int nchunks;
 122
 123         if (max_pages > MBPOOL_MAX_MAXPAGES || chunk_size == 0)
 124                 return (EINVAL);
 125         nchunks = page_size / chunk_size;
 126         if (nchunks == 0 || nchunks > MBPOOL_MAX_CHUNKS)
 127                 return (EINVAL);
 128
 129         (*pp) = malloc(sizeof(struct mbpool) +
 130             max_pages * sizeof(struct mbpage),
 131             M_MBPOOL, M_WAITOK | M_ZERO);
 132
 133         (*pp)->name = name;
 134         (*pp)->dmat = dmat;
 135         (*pp)->max_pages = max_pages;
 136         (*pp)->page_size = page_size;
 137         (*pp)->chunk_size = chunk_size;
 138         (*pp)->nchunks = nchunks;
 139
 140         SLIST_INIT(&(*pp)->free_list);
 141         mtx_init(&(*pp)->free_lock, name, NULL, MTX_DEF);
 142
 143         return (0);
 144 }
 145
 146 /*
 147  * destroy a pool
 148  */
 149 void
 150 mbp_destroy(struct mbpool *p)
 151 {
 152         u_int i;
 153         struct mbpage *pg;
 154 #ifdef DIAGNOSTIC
 155         struct mbtrail *tr;
 156         u_int b;
 157 #endif
 158
 159         for (i = 0; i < p->npages; i++) {
 160                 pg = &p->pages[i];
 161 #ifdef DIAGNOSTIC
 162                 for (b = 0; b < p->nchunks; b++) {
 163                         tr = C2T(p, N2C(p, pg, b));
 164                         if (tr->page & MBP_CARD)
 165                                 printf("%s: (%s) buf still on card"
 166                                     " %u/%u\n", __func__, p->name, i, b);
 167                         if (tr->page & MBP_USED)
 168                                 printf("%s: (%s) sbuf still in use"
 169                                     " %u/%u\n", __func__, p->name, i, b);
 170                 }
 171 #endif
 172                 bus_dmamap_unload(p->dmat, pg->map);
 173                 bus_dmamem_free(p->dmat, pg->va, pg->map);
 174         }
 175         mtx_destroy(&p->free_lock);
 176
 177         free(p, M_MBPOOL);
 178 }
 179
 180 /*
 181  * Helper function when loading a one segment DMA buffer.
 182  */
 183 static void
 184 mbp_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 185 {
 186         if (error == 0)
 187                 *(bus_addr_t *)arg = segs[0].ds_addr;
 188 }
 189
 190 /*
 191  * Allocate a new page
 192  */
 193 static void
 194 mbp_alloc_page(struct mbpool *p)
 195 {
 196         int error;
 197         struct mbpage *pg;
 198         u_int i;
 199         struct mbfree *f;
 200         struct mbtrail *t;
 201
 202         if (p->npages == p->max_pages) {
 203 #ifdef DIAGNOSTIC
 204                 printf("%s: (%s) page limit reached %u\n", __func__,
 205                     p->name, p->max_pages);
 206 #endif
 207                 return;
 208         }
 209         pg = &p->pages[p->npages];
 210
 211         error = bus_dmamem_alloc(p->dmat, &pg->va, BUS_DMA_NOWAIT, &pg->map);
 212         if (error != 0) {
 213                 free(pg, M_MBPOOL);
 214                 return;
 215         }
 216
 217         error = bus_dmamap_load(p->dmat, pg->map, pg->va, p->page_size,
 218             mbp_callback, &pg->phy, 0);
 219         if (error != 0) {
 220                 bus_dmamem_free(p->dmat, pg->va, pg->map);
 221                 free(pg, M_MBPOOL);
 222                 return;
 223         }
 224
 225         for (i = 0; i < p->nchunks; i++) {
 226                 f = N2C(p, pg, i);
 227                 t = C2T(p, f);
 228                 t->page = p->npages;
 229                 t->chunk = i;
 230                 SLIST_INSERT_HEAD(&p->free_list, f, link);
 231         }
 232
 233         p->npages++;
 234 }
 235
 236 /*
 237  * allocate a chunk
 238  */
 239 void *
 240 mbp_alloc(struct mbpool *p, bus_addr_t *pap, uint32_t *hp)
 241 {
 242         struct mbfree *cf;
 243         struct mbtrail *t;
 244
 245         mtx_lock(&p->free_lock);
 246         if ((cf = SLIST_FIRST(&p->free_list)) == NULL) {
 247                 mbp_alloc_page(p);
 248                 cf = SLIST_FIRST(&p->free_list);
 249         }
 250         if (cf == NULL) {
 251                 mtx_unlock(&p->free_lock);
 252                 return (NULL);
 253         }
 254         SLIST_REMOVE_HEAD(&p->free_list, link);
 255         mtx_unlock(&p->free_lock);
 256
 257         t = C2T(p, cf);
 258
 259         *pap = p->pages[t->page].phy + t->chunk * p->chunk_size;
 260         *hp = HMAKE(t->page, t->chunk);
 261
 262         t->page |= MBP_CARD | MBP_USED;
 263
 264         return (cf);
 265 }
 266
 267 /*
 268  * Free a chunk
 269  */
 270 void
 271 mbp_free(struct mbpool *p, void *ptr)
 272 {
 273         struct mbtrail *t;
 274
 275         mtx_lock(&p->free_lock);
 276         t = C2T(p, ptr);
 277         t->page &= ~(MBP_USED | MBP_CARD);
 278         SLIST_INSERT_HEAD(&p->free_list, (struct mbfree *)ptr, link);
 279         mtx_unlock(&p->free_lock);
 280 }
 281
 282 /*
 283  * Mbuf system external mbuf free routine
 284  */
 285 void
 286 mbp_ext_free(void *buf, void *arg)
 287 {
 288         mbp_free(arg, buf);
 289 }
 290
 291 /*
 292  * Free all buffers that are marked as beeing on the card
 293  */
 294 void
 295 mbp_card_free(struct mbpool *p)
 296 {
 297         u_int i, b;
 298         struct mbpage *pg;
 299         struct mbtrail *tr;
 300         struct mbfree *cf;
 301
 302         mtx_lock(&p->free_lock);
 303         for (i = 0; i < p->npages; i++) {
 304                 pg = &p->pages[i];
 305                 for (b = 0; b < p->nchunks; b++) {
 306                         cf = N2C(p, pg, b);
 307                         tr = C2T(p, cf);
 308                         if (tr->page & MBP_CARD) {
 309                                 tr->page &= MBP_PMSK;
 310                                 SLIST_INSERT_HEAD(&p->free_list, cf, link);
 311                         }
 312                 }
 313         }
 314         mtx_unlock(&p->free_lock);
 315 }
 316
 317 /*
 318  * Count buffers
 319  */
 320 void
 321 mbp_count(struct mbpool *p, u_int *used, u_int *card, u_int *free)
 322 {
 323         u_int i, b;
 324         struct mbpage *pg;
 325         struct mbtrail *tr;
 326         struct mbfree *cf;
 327
 328         *used = *card = *free = 0;
 329         for (i = 0; i < p->npages; i++) {
 330                 pg = &p->pages[i];
 331                 for (b = 0; b < p->nchunks; b++) {
 332                         tr = C2T(p, N2C(p, pg, b));
 333                         if (tr->page & MBP_CARD)
 334                                 (*card)++;
 335                         if (tr->page & MBP_USED)
 336                                 (*used)++;
 337                 }
 338         }
 339         mtx_lock(&p->free_lock);
 340         SLIST_FOREACH(cf, &p->free_list, link)
 341                 (*free)++;
 342         mtx_unlock(&p->free_lock);
 343 }
 344
 345 /*
 346  * Get the buffer from a handle and clear the card flag.
 347  */
 348 void *
 349 mbp_get(struct mbpool *p, uint32_t h)
 350 {
 351         struct mbfree *cf;
 352         struct mbtrail *tr;
 353
 354         cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h));
 355         tr = C2T(p, cf);
 356
 357 #ifdef DIAGNOSTIC
 358         if (!(tr->page & MBP_CARD))
 359                 printf("%s: (%s) chunk %u page %u not on card\n", __func__,
 360                     p->name, HCHUNK(h), HPAGE(h));
 361 #endif
 362
 363         tr->page &= ~MBP_CARD;
 364         return (cf);
 365 }
 366
 367 /*
 368  * Get the buffer from a handle and keep the card flag.
 369  */
 370 void *
 371 mbp_get_keep(struct mbpool *p, uint32_t h)
 372 {
 373         struct mbfree *cf;
 374         struct mbtrail *tr;
 375
 376         cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h));
 377         tr = C2T(p, cf);
 378
 379 #ifdef DIAGNOSTIC
 380         if (!(tr->page & MBP_CARD))
 381                 printf("%s: (%s) chunk %u page %u not on card\n", __func__,
 382                     p->name, HCHUNK(h), HPAGE(h));
 383 #endif
 384
 385         return (cf);
 386 }
 387
 388 /*
 389  * sync the chunk
 390  */
 391 void
 392 mbp_sync(struct mbpool *p, uint32_t h, bus_addr_t off, bus_size_t len, u_int op)
 393 {
 394
 395 #if 0
 396         bus_dmamap_sync_size(p->dmat, p->pages[HPAGE(h)].map,
 397             HCHUNK(h) * p->chunk_size + off, len, op);
 398 #endif
 399 }