release/src/shared/hnddma.c

   1 /*
   2  * Generic Broadcom Home Networking Division (HND) DMA module.
   3  * This supports the following chips: BCM42xx, 44xx, 47xx .
   4  *
   5  * Copyright 2005, Broadcom Corporation
   6  * All Rights Reserved.
   7  *
   8  * THIS SOFTWARE IS OFFERED "AS IS", AND BROADCOM GRANTS NO WARRANTIES OF ANY
   9  * KIND, EXPRESS OR IMPLIED, BY STATUTE, COMMUNICATION OR OTHERWISE. BROADCOM
  10  * SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
  11  * FOR A SPECIFIC PURPOSE OR NONINFRINGEMENT CONCERNING THIS SOFTWARE.
  12  *
  13  * $Id: hnddma.c,v 1.7 2005/03/07 08:35:32 kanki Exp $
  14  */
  15
  16 #include <typedefs.h>
  17 #include <osl.h>
  18 #include <bcmendian.h>
  19 #include <sbconfig.h>
  20 #include <bcmutils.h>
  21
  22 struct dma_info;        /* forward declaration */
  23 #define di_t struct dma_info
  24 #include <hnddma.h>
  25
  26 /* debug/trace */
  27 #define DMA_ERROR(args)
  28 #define DMA_TRACE(args)
  29
  30 /* default dma message level(if input msg_level pointer is null in dma_attach()) */
  31 static uint dma_msg_level = 0;
  32
  33 #define MAXNAMEL        8
  34 #define MAXDD           (DMAMAXRINGSZ / sizeof (dmadd_t))
  35
  36 /* dma engine software state */
  37 typedef struct dma_info {
  38         hnddma_t        hnddma;         /* exported structure */
  39         uint            *msg_level;     /* message level pointer */
  40
  41         char            name[MAXNAMEL]; /* callers name for diag msgs */
  42         void            *drv;           /* driver handle */
  43         void            *osh;           /* os handle */
  44         dmaregs_t       *regs;          /* dma engine registers */
  45
  46         dmadd_t         *txd;           /* pointer to chip-specific tx descriptor ring */
  47         uint            txin;           /* index of next descriptor to reclaim */
  48         uint            txout;          /* index of next descriptor to post */
  49         uint            txavail;        /* # free tx descriptors */
  50         void            **txp;          /* pointer to parallel array of pointers to packets */
  51         ulong           txdpa;          /* physical address of descriptor ring */
  52         uint            txdalign;       /* #bytes added to alloc'd mem to align txd */
  53         uint            txdalloc;       /* #bytes allocated for the ring */
  54
  55         dmadd_t         *rxd;           /* pointer to chip-specific rx descriptor ring */
  56         uint            rxin;           /* index of next descriptor to reclaim */
  57         uint            rxout;          /* index of next descriptor to post */
  58         void            **rxp;          /* pointer to parallel array of pointers to packets */
  59         ulong           rxdpa;          /* physical address of descriptor ring */
  60         uint            rxdalign;       /* #bytes added to alloc'd mem to align rxd */
  61         uint            rxdalloc;       /* #bytes allocated for the ring */
  62
  63         /* tunables */
  64         uint            ntxd;           /* # tx descriptors */
  65         uint            nrxd;           /* # rx descriptors */
  66         uint            rxbufsize;      /* rx buffer size in bytes */
  67         uint            nrxpost;        /* # rx buffers to keep posted */
  68         uint            rxoffset;       /* rxcontrol offset */
  69         uint            ddoffset;       /* add to get dma address of descriptor ring */
  70         uint            dataoffset;     /* add to get dma address of data buffer */
  71 } dma_info_t;
  72
  73 /* descriptor bumping macros */
  74 #define XXD(x, n)       ((x) & ((n) - 1))
  75 #define TXD(x)          XXD((x), di->ntxd)
  76 #define RXD(x)          XXD((x), di->nrxd)
  77 #define NEXTTXD(i)      TXD(i + 1)
  78 #define PREVTXD(i)      TXD(i - 1)
  79 #define NEXTRXD(i)      RXD(i + 1)
  80 #define NTXDACTIVE(h, t)        TXD(t - h)
  81 #define NRXDACTIVE(h, t)        RXD(t - h)
  82
  83 /* macros to convert between byte offsets and indexes */
  84 #define B2I(bytes)      ((bytes) / sizeof (dmadd_t))
  85 #define I2B(index)      ((index) * sizeof (dmadd_t))
  86
  87 /*
  88  * This assume the largest i/o address is, in fact, the pci big window
  89  * and that the pci core sb2pcitranslation2 register has been left with
  90  * the default 0x0 pci base address.
  91  */
  92 #define MAXDMAADDR              SB_PCI_DMA_SZ
  93 #define DMA_ADDRESSABLE(x)      !((x) & ~(MAXDMAADDR - 1))
  94
  95 /* prototypes */
  96
  97 void*
  98 dma_attach(void *drv, void *osh, char *name, dmaregs_t *regs, uint ntxd, uint nrxd,
  99         uint rxbufsize, uint nrxpost, uint rxoffset, uint ddoffset, uint dataoffset, uint *msg_level)
 100 {
 101         dma_info_t *di;
 102         uint size;
 103         void *va;
 104
 105         ASSERT(ntxd <= MAXDD);
 106         ASSERT(ISPOWEROF2(ntxd));
 107         ASSERT(nrxd <= MAXDD);
 108         ASSERT(ISPOWEROF2(nrxd));
 109
 110         /* allocate private info structure */
 111         if ((di = MALLOC(osh, sizeof (dma_info_t))) == NULL) {
 112                 return (NULL);
 113         }
 114         bzero((char*)di, sizeof (dma_info_t));
 115
 116         /* allocate tx packet pointer vector */
 117         if (ntxd) {
 118                 size = ntxd * sizeof (void*);
 119                 if ((di->txp = MALLOC(osh, size)) == NULL)
 120                         goto fail;
 121                 bzero((char*)di->txp, size);
 122         }
 123
 124         /* allocate rx packet pointer vector */
 125         if (nrxd) {
 126                 size = nrxd * sizeof (void*);
 127                 if ((di->rxp = MALLOC(osh, size)) == NULL)
 128                         goto fail;
 129                 bzero((char*)di->rxp, size);
 130         }
 131
 132         /* set message level */
 133         di->msg_level = msg_level ? msg_level : &dma_msg_level;
 134
 135         DMA_TRACE(("%s: dma_attach: drv %p osh %p regs %p ntxd %d nrxd %d rxbufsize %d nrxpost %d rxoffset %d ddoffset 0x%x dataoffset 0x%x\n", name, drv, osh, regs, ntxd, nrxd, rxbufsize, nrxpost, rxoffset, ddoffset, dataoffset));
 136
 137         /* make a private copy of our callers name */
 138         strncpy(di->name, name, MAXNAMEL);
 139         di->name[MAXNAMEL-1] = '\0';
 140
 141         di->drv = drv;
 142         di->osh = osh;
 143         di->regs = regs;
 144
 145         /* allocate transmit descriptor ring */
 146         if (ntxd) {
 147                 /* only need ntxd descriptors but it must be DMARINGALIGNed */
 148                 size = ntxd * sizeof (dmadd_t);
 149                 if (!ISALIGNED(DMA_CONSISTENT_ALIGN, DMARINGALIGN))
 150                         size += DMARINGALIGN;
 151                 if ((va = DMA_ALLOC_CONSISTENT(osh, size, &di->txdpa)) == NULL)
 152                         goto fail;
 153                 di->txd = (dmadd_t*) ROUNDUP((uintptr)va, DMARINGALIGN);
 154                 di->txdalign = (uint)((int8*)di->txd - (int8*)va);
 155                 di->txdpa += di->txdalign;
 156                 di->txdalloc = size;
 157                 ASSERT(ISALIGNED((uintptr)di->txd, DMARINGALIGN));
 158                 ASSERT(DMA_ADDRESSABLE(di->txdpa));
 159         }
 160
 161         /* allocate receive descriptor ring */
 162         if (nrxd) {
 163                 /* only need nrxd descriptors but it must be DMARINGALIGNed */
 164                 size = nrxd * sizeof (dmadd_t);
 165                 if (!ISALIGNED(DMA_CONSISTENT_ALIGN, DMARINGALIGN))
 166                         size += DMARINGALIGN;
 167                 if ((va = DMA_ALLOC_CONSISTENT(osh, size, &di->rxdpa)) == NULL)
 168                         goto fail;
 169                 di->rxd = (dmadd_t*) ROUNDUP((uintptr)va, DMARINGALIGN);
 170                 di->rxdalign = (uint)((int8*)di->rxd - (int8*)va);
 171                 di->rxdpa += di->rxdalign;
 172                 di->rxdalloc = size;
 173                 ASSERT(ISALIGNED((uintptr)di->rxd, DMARINGALIGN));
 174                 ASSERT(DMA_ADDRESSABLE(di->rxdpa));
 175         }
 176
 177         /* save tunables */
 178         di->ntxd = ntxd;
 179         di->nrxd = nrxd;
 180         di->rxbufsize = rxbufsize;
 181         di->nrxpost = nrxpost;
 182         di->rxoffset = rxoffset;
 183         di->ddoffset = ddoffset;
 184         di->dataoffset = dataoffset;
 185
 186         return ((void*)di);
 187
 188 fail:
 189         dma_detach((void*)di);
 190         return (NULL);
 191 }
 192
 193 /* may be called with core in reset */
 194 void
 195 dma_detach(dma_info_t *di)
 196 {
 197         if (di == NULL)
 198                 return;
 199
 200         DMA_TRACE(("%s: dma_detach\n", di->name));
 201
 202         /* shouldn't be here if descriptors are unreclaimed */
 203         ASSERT(di->txin == di->txout);
 204         ASSERT(di->rxin == di->rxout);
 205
 206         /* free dma descriptor rings */
 207         if (di->txd)
 208                 DMA_FREE_CONSISTENT(di->osh, ((int8*)di->txd - di->txdalign),
 209                         di->txdalloc, (di->txdpa - di->txdalign));
 210         if (di->rxd)
 211                 DMA_FREE_CONSISTENT(di->osh, ((int8*)di->rxd - di->rxdalign),
 212                         di->rxdalloc, (di->rxdpa - di->rxdalign));
 213
 214         /* free packet pointer vectors */
 215         if (di->txp)
 216                 MFREE(di->osh, (void*)di->txp, (di->ntxd * sizeof (void*)));
 217         if (di->rxp)
 218                 MFREE(di->osh, (void*)di->rxp, (di->nrxd * sizeof (void*)));
 219
 220         /* free our private info structure */
 221         MFREE(di->osh, (void*)di, sizeof (dma_info_t));
 222 }
 223
 224
 225 void
 226 dma_txreset(dma_info_t *di)
 227 {
 228         uint32 status;
 229
 230         DMA_TRACE(("%s: dma_txreset\n", di->name));
 231
 232         /* suspend tx DMA first */
 233         W_REG(&di->regs->xmtcontrol, XC_SE);
 234         SPINWAIT((status = (R_REG(&di->regs->xmtstatus) & XS_XS_MASK)) != XS_XS_DISABLED &&
 235                  status != XS_XS_IDLE &&
 236                  status != XS_XS_STOPPED,
 237                  10000);
 238
 239         W_REG(&di->regs->xmtcontrol, 0);
 240         SPINWAIT((status = (R_REG(&di->regs->xmtstatus) & XS_XS_MASK)) != XS_XS_DISABLED,
 241                  10000);
 242
 243         if (status != XS_XS_DISABLED) {
 244                 DMA_ERROR(("%s: dma_txreset: dma cannot be stopped\n", di->name));
 245         }
 246
 247         /* wait for the last transaction to complete */
 248         OSL_DELAY(300);
 249 }
 250
 251 void
 252 dma_rxreset(dma_info_t *di)
 253 {
 254         uint32 status;
 255
 256         DMA_TRACE(("%s: dma_rxreset\n", di->name));
 257
 258         W_REG(&di->regs->rcvcontrol, 0);
 259         SPINWAIT((status = (R_REG(&di->regs->rcvstatus) & RS_RS_MASK)) != RS_RS_DISABLED,
 260                  10000);
 261
 262         if (status != RS_RS_DISABLED) {
 263                 DMA_ERROR(("%s: dma_rxreset: dma cannot be stopped\n", di->name));
 264         }
 265 }
 266
 267 void
 268 dma_txinit(dma_info_t *di)
 269 {
 270         DMA_TRACE(("%s: dma_txinit\n", di->name));
 271
 272         di->txin = di->txout = 0;
 273         di->txavail = di->ntxd - 1;
 274
 275         /* clear tx descriptor ring */
 276         BZERO_SM((void*)di->txd, (di->ntxd * sizeof (dmadd_t)));
 277
 278         W_REG(&di->regs->xmtcontrol, XC_XE);
 279         W_REG(&di->regs->xmtaddr, (di->txdpa + di->ddoffset));
 280 }
 281
 282 bool
 283 dma_txenabled(dma_info_t *di)
 284 {
 285         uint32 xc;
 286
 287         /* If the chip is dead, it is not enabled :-) */
 288         xc = R_REG(&di->regs->xmtcontrol);
 289         return ((xc != 0xffffffff) && (xc & XC_XE));
 290 }
 291
 292 void
 293 dma_txsuspend(dma_info_t *di)
 294 {
 295         DMA_TRACE(("%s: dma_txsuspend\n", di->name));
 296         OR_REG(&di->regs->xmtcontrol, XC_SE);
 297 }
 298
 299 void
 300 dma_txresume(dma_info_t *di)
 301 {
 302         DMA_TRACE(("%s: dma_txresume\n", di->name));
 303         AND_REG(&di->regs->xmtcontrol, ~XC_SE);
 304 }
 305
 306 bool
 307 dma_txsuspended(dma_info_t *di)
 308 {
 309         if (!(R_REG(&di->regs->xmtcontrol) & XC_SE))
 310                 return 0;
 311
 312         if ((R_REG(&di->regs->xmtstatus) & XS_XS_MASK) != XS_XS_IDLE)
 313                 return 0;
 314
 315         OSL_DELAY(2);
 316         return ((R_REG(&di->regs->xmtstatus) & XS_XS_MASK) == XS_XS_IDLE);
 317 }
 318
 319 bool
 320 dma_txstopped(dma_info_t *di)
 321 {
 322         return ((R_REG(&di->regs->xmtstatus) & XS_XS_MASK) == XS_XS_STOPPED);
 323 }
 324
 325 bool
 326 dma_rxstopped(dma_info_t *di)
 327 {
 328         return ((R_REG(&di->regs->rcvstatus) & RS_RS_MASK) == RS_RS_STOPPED);
 329 }
 330
 331 void
 332 dma_fifoloopbackenable(dma_info_t *di)
 333 {
 334         DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name));
 335         OR_REG(&di->regs->xmtcontrol, XC_LE);
 336 }
 337
 338 void
 339 dma_rxinit(dma_info_t *di)
 340 {
 341         DMA_TRACE(("%s: dma_rxinit\n", di->name));
 342
 343         di->rxin = di->rxout = 0;
 344
 345         /* clear rx descriptor ring */
 346         BZERO_SM((void*)di->rxd, (di->nrxd * sizeof (dmadd_t)));
 347
 348         dma_rxenable(di);
 349         W_REG(&di->regs->rcvaddr, (di->rxdpa + di->ddoffset));
 350 }
 351
 352 void
 353 dma_rxenable(dma_info_t *di)
 354 {
 355         DMA_TRACE(("%s: dma_rxenable\n", di->name));
 356         W_REG(&di->regs->rcvcontrol, ((di->rxoffset << RC_RO_SHIFT) | RC_RE));
 357 }
 358
 359 bool
 360 dma_rxenabled(dma_info_t *di)
 361 {
 362         uint32 rc;
 363
 364         rc = R_REG(&di->regs->rcvcontrol);
 365         return ((rc != 0xffffffff) && (rc & RC_RE));
 366 }
 367
 368 /*
 369  * The BCM47XX family supports full 32bit dma engine buffer addressing so
 370  * dma buffers can cross 4 Kbyte page boundaries.
 371  */
 372 int
 373 dma_txfast(dma_info_t *di, void *p0, uint32 coreflags)
 374 {
 375         void *p, *next;
 376         uchar *data;
 377         uint len;
 378         uint txout;
 379         uint32 ctrl;
 380         uint32 pa;
 381
 382         DMA_TRACE(("%s: dma_txfast\n", di->name));
 383
 384         txout = di->txout;
 385         ctrl = 0;
 386
 387         /*
 388          * Walk the chain of packet buffers
 389          * allocating and initializing transmit descriptor entries.
 390          */
 391         for (p = p0; p; p = next) {
 392                 data = PKTDATA(di->drv, p);
 393                 len = PKTLEN(di->drv, p);
 394                 next = PKTNEXT(di->drv, p);
 395
 396                 /* return nonzero if out of tx descriptors */
 397                 if (NEXTTXD(txout) == di->txin)
 398                         goto outoftxd;
 399
 400                 if (len == 0)
 401                         continue;
 402
 403                 /* get physical address of buffer start */
 404                 pa = (uint32) DMA_MAP(di->osh, data, len, DMA_TX, p);
 405                 ASSERT(DMA_ADDRESSABLE(pa));
 406
 407                 /* build the descriptor control value */
 408                 ctrl = len & CTRL_BC_MASK;
 409
 410                 ctrl |= coreflags;
 411
 412                 if (p == p0)
 413                         ctrl |= CTRL_SOF;
 414                 if (next == NULL)
 415                         ctrl |= (CTRL_IOC | CTRL_EOF);
 416                 if (txout == (di->ntxd - 1))
 417                         ctrl |= CTRL_EOT;
 418
 419                 /* init the tx descriptor */
 420                 W_SM(&di->txd[txout].ctrl, BUS_SWAP32(ctrl));
 421                 W_SM(&di->txd[txout].addr, BUS_SWAP32(pa + di->dataoffset));
 422
 423                 ASSERT(di->txp[txout] == NULL);
 424
 425                 txout = NEXTTXD(txout);
 426         }
 427
 428         /* if last txd eof not set, fix it */
 429         if (!(ctrl & CTRL_EOF))
 430                 W_SM(&di->txd[PREVTXD(txout)].ctrl, BUS_SWAP32(ctrl | CTRL_IOC | CTRL_EOF));
 431
 432         /* save the packet */
 433         di->txp[PREVTXD(txout)] = p0;
 434
 435         /* bump the tx descriptor index */
 436         di->txout = txout;
 437
 438         /* kick the chip */
 439         W_REG(&di->regs->xmtptr, I2B(txout));
 440
 441         /* tx flow control */
 442         di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
 443
 444         return (0);
 445
 446 outoftxd:
 447         DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name));
 448         PKTFREE(di->drv, p0, TRUE);
 449         di->txavail = 0;
 450         di->hnddma.txnobuf++;
 451         return (-1);
 452 }
 453
 454 #define PAGESZ          4096
 455 #define PAGEBASE(x)     ((uintptr)(x) & ~4095)
 456
 457 /*
 458  * Just like above except go through the extra effort of splitting
 459  * buffers that cross 4Kbyte boundaries into multiple tx descriptors.
 460  */
 461 int
 462 dma_tx(dma_info_t *di, void *p0, uint32 coreflags)
 463 {
 464         void *p, *next;
 465         uchar *data;
 466         uint plen, len;
 467         uchar *page, *start, *end;
 468         uint txout;
 469         uint32 ctrl;
 470         uint32 pa;
 471
 472         DMA_TRACE(("%s: dma_tx\n", di->name));
 473
 474         txout = di->txout;
 475         ctrl = 0;
 476
 477         /*
 478          * Walk the chain of packet buffers
 479          * splitting those that cross 4 Kbyte boundaries
 480          * allocating and initializing transmit descriptor entries.
 481          */
 482         for (p = p0; p; p = next) {
 483                 data = PKTDATA(di->drv, p);
 484                 plen = PKTLEN(di->drv, p);
 485                 next = PKTNEXT(di->drv, p);
 486
 487                 if (plen == 0)
 488                         continue;
 489
 490                 for (page = (uchar*)PAGEBASE(data);
 491                         page <= (uchar*)PAGEBASE(data + plen - 1);
 492                         page += PAGESZ) {
 493
 494                         /* return nonzero if out of tx descriptors */
 495                         if (NEXTTXD(txout) == di->txin)
 496                                 goto outoftxd;
 497
 498                         start = (page == (uchar*)PAGEBASE(data))?  data: page;
 499                         end = (page == (uchar*)PAGEBASE(data + plen))?
 500                                 (data + plen): (page + PAGESZ);
 501                         len = (uint)(end - start);
 502
 503                         /* build the descriptor control value */
 504                         ctrl = len & CTRL_BC_MASK;
 505
 506                         ctrl |= coreflags;
 507
 508                         if ((p == p0) && (start == data))
 509                                 ctrl |= CTRL_SOF;
 510                         if ((next == NULL) && (end == (data + plen)))
 511                                 ctrl |= (CTRL_IOC | CTRL_EOF);
 512                         if (txout == (di->ntxd - 1))
 513                                 ctrl |= CTRL_EOT;
 514
 515                         /* get physical address of buffer start */
 516                         pa = (uint32) DMA_MAP(di->osh, start, len, DMA_TX, p);
 517                         ASSERT(DMA_ADDRESSABLE(pa));
 518
 519                         /* init the tx descriptor */
 520                         W_SM(&di->txd[txout].ctrl, BUS_SWAP32(ctrl));
 521                         W_SM(&di->txd[txout].addr, BUS_SWAP32(pa + di->dataoffset));
 522
 523                         ASSERT(di->txp[txout] == NULL);
 524
 525                         txout = NEXTTXD(txout);
 526                 }
 527         }
 528
 529         /* if last txd eof not set, fix it */
 530         if (!(ctrl & CTRL_EOF))
 531                 W_SM(&di->txd[PREVTXD(txout)].ctrl, BUS_SWAP32(ctrl | CTRL_IOC | CTRL_EOF));
 532
 533         /* save the packet */
 534         di->txp[PREVTXD(txout)] = p0;
 535
 536         /* bump the tx descriptor index */
 537         di->txout = txout;
 538
 539         /* kick the chip */
 540         W_REG(&di->regs->xmtptr, I2B(txout));
 541
 542         /* tx flow control */
 543         di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
 544
 545         return (0);
 546
 547 outoftxd:
 548         DMA_ERROR(("%s: dma_tx: out of txds\n", di->name));
 549         PKTFREE(di->drv, p0, TRUE);
 550         di->txavail = 0;
 551         di->hnddma.txnobuf++;
 552         return (-1);
 553 }
 554
 555 /* returns a pointer to the next frame received, or NULL if there are no more */
 556 void*
 557 dma_rx(dma_info_t *di)
 558 {
 559         void *p;
 560         uint len;
 561         int skiplen = 0;
 562
 563         while ((p = dma_getnextrxp(di, FALSE))) {
 564                 /* skip giant packets which span multiple rx descriptors */
 565                 if (skiplen > 0) {
 566                         skiplen -= di->rxbufsize;
 567                         if (skiplen < 0)
 568                                 skiplen = 0;
 569                         PKTFREE(di->drv, p, FALSE);
 570                         continue;
 571                 }
 572
 573                 len = ltoh16(*(uint16*)(PKTDATA(di->drv, p)));
 574                 DMA_TRACE(("%s: dma_rx len %d\n", di->name, len));
 575
 576                 /* bad frame length check */
 577                 if (len > (di->rxbufsize - di->rxoffset)) {
 578                         DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n", di->name, len));
 579                         if (len > 0)
 580                                 skiplen = len - (di->rxbufsize - di->rxoffset);
 581                         PKTFREE(di->drv, p, FALSE);
 582                         di->hnddma.rxgiants++;
 583                         continue;
 584                 }
 585
 586                 /* set actual length */
 587                 PKTSETLEN(di->drv, p, (di->rxoffset + len));
 588
 589                 break;
 590         }
 591
 592         return (p);
 593 }
 594
 595 /* post receive buffers */
 596 void
 597 dma_rxfill(dma_info_t *di)
 598 {
 599         void *p;
 600         uint rxin, rxout;
 601         uint ctrl;
 602         uint n;
 603         uint i;
 604         uint32 pa;
 605         uint rxbufsize;
 606
 607         /*
 608          * Determine how many receive buffers we're lacking
 609          * from the full complement, allocate, initialize,
 610          * and post them, then update the chip rx lastdscr.
 611          */
 612
 613         rxin = di->rxin;
 614         rxout = di->rxout;
 615         rxbufsize = di->rxbufsize;
 616
 617         n = di->nrxpost - NRXDACTIVE(rxin, rxout);
 618
 619         DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n));
 620
 621         for (i = 0; i < n; i++) {
 622                 if ((p = PKTGET(di->drv, rxbufsize, FALSE)) == NULL) {
 623                         DMA_ERROR(("%s: dma_rxfill: out of rxbufs\n", di->name));
 624                         di->hnddma.rxnobuf++;
 625                         break;
 626                 }
 627
 628                 *(uint32*)(OSL_UNCACHED(PKTDATA(di->drv, p))) = 0;
 629
 630                 pa = (uint32) DMA_MAP(di->osh, PKTDATA(di->drv, p), rxbufsize, DMA_RX, p);
 631                 ASSERT(ISALIGNED(pa, 4));
 632                 ASSERT(DMA_ADDRESSABLE(pa));
 633
 634                 /* save the free packet pointer */
 635                 ASSERT(di->rxp[rxout] == NULL);
 636                 di->rxp[rxout] = p;
 637
 638                 /* prep the descriptor control value */
 639                 ctrl = rxbufsize;
 640                 if (rxout == (di->nrxd - 1))
 641                         ctrl |= CTRL_EOT;
 642
 643                 /* init the rx descriptor */
 644                 W_SM(&di->rxd[rxout].ctrl, BUS_SWAP32(ctrl));
 645                 W_SM(&di->rxd[rxout].addr, BUS_SWAP32(pa + di->dataoffset));
 646
 647                 rxout = NEXTRXD(rxout);
 648         }
 649
 650         di->rxout = rxout;
 651
 652         /* update the chip lastdscr pointer */
 653         W_REG(&di->regs->rcvptr, I2B(rxout));
 654 }
 655
 656 void
 657 dma_txreclaim(dma_info_t *di, bool forceall)
 658 {
 659         void *p;
 660
 661         DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, forceall ? "all" : ""));
 662
 663         while ((p = dma_getnexttxp(di, forceall)))
 664                 PKTFREE(di->drv, p, TRUE);
 665 }
 666
 667 /*
 668  * Reclaim next completed txd (txds if using chained buffers) and
 669  * return associated packet.
 670  * If 'force' is true, reclaim txd(s) and return associated packet
 671  * regardless of the value of the hardware "curr" pointer.
 672  */
 673 void*
 674 dma_getnexttxp(dma_info_t *di, bool forceall)
 675 {
 676         uint start, end, i;
 677         void *txp;
 678
 679         DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, forceall ? "all" : ""));
 680
 681         txp = NULL;
 682
 683         start = di->txin;
 684         if (forceall)
 685                 end = di->txout;
 686         else
 687                 end = B2I(R_REG(&di->regs->xmtstatus) & XS_CD_MASK);
 688
 689         if ((start == 0) && (end > di->txout))
 690                 goto bogus;
 691
 692         for (i = start; i != end && !txp; i = NEXTTXD(i)) {
 693                 DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->txd[i].addr)) - di->dataoffset),
 694                           (BUS_SWAP32(R_SM(&di->txd[i].ctrl)) & CTRL_BC_MASK), DMA_TX, di->txp[i]);
 695                 W_SM(&di->txd[i].addr, 0xdeadbeef);
 696                 txp = di->txp[i];
 697                 di->txp[i] = NULL;
 698         }
 699
 700         di->txin = i;
 701
 702         /* tx flow control */
 703         di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
 704
 705         return (txp);
 706
 707 bogus:
 708 /*
 709         DMA_ERROR(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
 710                 start, end, di->txout, forceall));
 711 */
 712         return (NULL);
 713 }
 714
 715 /* like getnexttxp but no reclaim */
 716 void*
 717 dma_peeknexttxp(dma_info_t *di)
 718 {
 719         uint end, i;
 720
 721         end = B2I(R_REG(&di->regs->xmtstatus) & XS_CD_MASK);
 722
 723         for (i = di->txin; i != end; i = NEXTTXD(i))
 724                 if (di->txp[i])
 725                         return (di->txp[i]);
 726
 727         return (NULL);
 728 }
 729
 730 void
 731 dma_rxreclaim(dma_info_t *di)
 732 {
 733         void *p;
 734
 735         DMA_TRACE(("%s: dma_rxreclaim\n", di->name));
 736
 737         while ((p = dma_getnextrxp(di, TRUE)))
 738                 PKTFREE(di->drv, p, FALSE);
 739 }
 740
 741 void *
 742 dma_getnextrxp(dma_info_t *di, bool forceall)
 743 {
 744         uint i;
 745         void *rxp;
 746
 747         /* if forcing, dma engine must be disabled */
 748         ASSERT(!forceall || !dma_rxenabled(di));
 749
 750         i = di->rxin;
 751
 752         /* return if no packets posted */
 753         if (i == di->rxout)
 754                 return (NULL);
 755
 756         /* ignore curr if forceall */
 757         if (!forceall && (i == B2I(R_REG(&di->regs->rcvstatus) & RS_CD_MASK)))
 758                 return (NULL);
 759
 760         /* get the packet pointer that corresponds to the rx descriptor */
 761         rxp = di->rxp[i];
 762         ASSERT(rxp);
 763         di->rxp[i] = NULL;
 764
 765         /* clear this packet from the descriptor ring */
 766         DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->rxd[i].addr)) - di->dataoffset),
 767                   di->rxbufsize, DMA_RX, rxp);
 768         W_SM(&di->rxd[i].addr, 0xdeadbeef);
 769
 770         di->rxin = NEXTRXD(i);
 771
 772         return (rxp);
 773 }
 774
 775
 776 uintptr
 777 dma_getvar(dma_info_t *di, char *name)
 778 {
 779         if (!strcmp(name, "&txavail"))
 780                 return ((uintptr) &di->txavail);
 781         else {
 782                 ASSERT(0);
 783         }
 784         return (0);
 785 }
 786
 787 void
 788 dma_txblock(dma_info_t *di)
 789 {
 790         di->txavail = 0;
 791 }
 792
 793 void
 794 dma_txunblock(dma_info_t *di)
 795 {
 796         di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
 797 }
 798
 799 uint
 800 dma_txactive(dma_info_t *di)
 801 {
 802         return (NTXDACTIVE(di->txin, di->txout));
 803 }
 804
 805 /*
 806  * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
 807  */
 808 void
 809 dma_txrotate(di_t *di)
 810 {
 811         uint ad;
 812         uint nactive;
 813         uint rot;
 814         uint old, new;
 815         uint32 w;
 816         uint first, last;
 817
 818         ASSERT(dma_txsuspended(di));
 819
 820         nactive = dma_txactive(di);
 821         ad = B2I((R_REG(&di->regs->xmtstatus) & XS_AD_MASK) >> XS_AD_SHIFT);
 822         rot = TXD(ad - di->txin);
 823
 824         ASSERT(rot < di->ntxd);
 825
 826         /* full-ring case is a lot harder - don't worry about this */
 827         if (rot >= (di->ntxd - nactive)) {
 828                 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
 829                 return;
 830         }
 831
 832         first = di->txin;
 833         last = PREVTXD(di->txout);
 834
 835         /* move entries starting at last and moving backwards to first */
 836         for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
 837                 new = TXD(old + rot);
 838
 839                 /*
 840                  * Move the tx dma descriptor.
 841                  * EOT is set only in the last entry in the ring.
 842                  */
 843                 w = R_SM(&di->txd[old].ctrl) & ~CTRL_EOT;
 844                 if (new == (di->ntxd - 1))
 845                         w |= CTRL_EOT;
 846                 W_SM(&di->txd[new].ctrl, w);
 847                 W_SM(&di->txd[new].addr, R_SM(&di->txd[old].addr));
 848
 849                 /* zap the old tx dma descriptor address field */
 850                 W_SM(&di->txd[old].addr, 0xdeadbeef);
 851
 852                 /* move the corresponding txp[] entry */
 853                 ASSERT(di->txp[new] == NULL);
 854                 di->txp[new] = di->txp[old];
 855                 di->txp[old] = NULL;
 856         }
 857
 858         /* update txin and txout */
 859         di->txin = ad;
 860         di->txout = TXD(di->txout + rot);
 861         di->txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
 862
 863         /* kick the chip */
 864         W_REG(&di->regs->xmtptr, I2B(di->txout));
 865 }