?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
9484 cxgbe should clean TX descriptors in timely manner
[unleashed.git] / usr / src / uts / common / io / cxgbe / t4nex / t4_sge.c
blob2109c6fcf2179d6cbcd9b8e36a7fc2ed6c5f6401
1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/atomic.h>
27 #include <sys/dlpi.h>
28 #include <sys/pattr.h>
29 #include <sys/strsubr.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <inet/ip.h>
33 #include <inet/tcp.h>
34
35 #include "version.h"
36 #include "common/common.h"
37 #include "common/t4_msg.h"
38 #include "common/t4_regs.h"
39 #include "common/t4_regs_values.h"
40
41 /* TODO: Tune. */
42 int rx_buf_size = 8192;
43 int tx_copy_threshold = 256;
44 uint16_t rx_copy_threshold = 256;
45
46 /* Used to track coalesced tx work request */
47 struct txpkts {
48 mblk_t *tail; /* head is in the software descriptor */
49 uint64_t *flitp; /* ptr to flit where next pkt should start */
50 uint8_t npkt; /* # of packets in this work request */
51 uint8_t nflits; /* # of flits used by this work request */
52 uint16_t plen; /* total payload (sum of all packets) */
53 };
54
55 /* All information needed to tx a frame */
56 struct txinfo {
57 uint32_t len; /* Total length of frame */
58 uint32_t flags; /* Checksum and LSO flags */
59 uint32_t mss; /* MSS for LSO */
60 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */
61 uint8_t nflits; /* # of flits needed for the SGL */
62 uint8_t hdls_used; /* # of DMA handles used */
63 uint32_t txb_used; /* txb_space used */
64 struct ulptx_sgl sgl __attribute__((aligned(8)));
65 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
66 };
67
68 static int service_iq(struct sge_iq *iq, int budget);
69 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
70 int8_t pktc_idx, int qsize, uint8_t esize);
71 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
72 static inline void init_eq(struct adapter *sc, struct sge_eq *eq,
73 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid);
74 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
75 struct sge_fl *fl, int intr_idx, int cong);
76 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
77 struct sge_fl *fl);
78 static int alloc_fwq(struct adapter *sc);
79 static int free_fwq(struct adapter *sc);
80 #ifdef TCP_OFFLOAD_ENABLE
81 static int alloc_mgmtq(struct adapter *sc);
82 #endif
83 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
84 int i);
85 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
86 #ifdef TCP_OFFLOAD_ENABLE
87 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
88 int intr_idx);
89 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq);
90 #endif
91 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq);
92 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
93 struct sge_eq *eq);
94 #ifdef TCP_OFFLOAD_ENABLE
95 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi,
96 struct sge_eq *eq);
97 #endif
98 static int alloc_eq(struct adapter *sc, struct port_info *pi,
99 struct sge_eq *eq);
100 static int free_eq(struct adapter *sc, struct sge_eq *eq);
101 #ifdef TCP_OFFLOAD_ENABLE
102 static int alloc_wrq(struct adapter *sc, struct port_info *pi,
103 struct sge_wrq *wrq, int idx);
104 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq);
105 #endif
106 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
107 static int free_txq(struct port_info *pi, struct sge_txq *txq);
108 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
109 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
110 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
111 caddr_t *pva);
112 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
113 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
114 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
115 caddr_t *pva);
116 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
117 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
118 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
119 caddr_t *pva);
120 static inline bool is_new_response(const struct sge_iq *iq,
121 struct rsp_ctrl **ctrl);
122 static inline void iq_next(struct sge_iq *iq);
123 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
124 static void refill_sfl(void *arg);
125 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
126 static void free_fl_bufs(struct sge_fl *fl);
127 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl,
128 uint32_t len_newbuf, int *fl_bufs_used);
129 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
130 struct txinfo *txinfo, int sgl_only);
131 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
132 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
133 struct txinfo *txinfo);
134 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
135 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
136 mblk_t *m, int len);
137 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
138 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
139 struct txinfo *txinfo);
140 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
141 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
142 struct txinfo *txinfo);
143 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
144 struct txpkts *txpkts, struct txinfo *txinfo);
145 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
146 int len);
147 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
148 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
149 static void write_txqflush_wr(struct sge_txq *txq);
150 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
151 mblk_t *m);
152 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
153 static kstat_t *setup_port_config_kstats(struct port_info *pi);
154 static kstat_t *setup_port_info_kstats(struct port_info *pi);
155 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
156 int idx);
157 static int update_rxq_kstats(kstat_t *ksp, int rw);
158 static int update_port_info_kstats(kstat_t *ksp, int rw);
159 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
160 int idx);
161 static int update_txq_kstats(kstat_t *ksp, int rw);
162 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
163 mblk_t *);
164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss,
165 mblk_t *m);
166
167 static inline int
168 reclaimable(struct sge_eq *eq)
169 {
170 unsigned int cidx;
171
172 cidx = eq->spg->cidx; /* stable snapshot */
173 cidx = be16_to_cpu(cidx);
174
175 if (cidx >= eq->cidx)
176 return (cidx - eq->cidx);
177 else
178 return (cidx + eq->cap - eq->cidx);
179 }
180
181 void
182 t4_sge_init(struct adapter *sc)
183 {
184 struct driver_properties *p = &sc->props;
185 ddi_dma_attr_t *dma_attr;
186 ddi_device_acc_attr_t *acc_attr;
187 uint32_t sge_control, sge_conm_ctrl;
188 int egress_threshold;
189
190 /*
191 * Device access and DMA attributes for descriptor rings
192 */
193 acc_attr = &sc->sge.acc_attr_desc;
194 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
195 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
196 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
197
198 dma_attr = &sc->sge.dma_attr_desc;
199 dma_attr->dma_attr_version = DMA_ATTR_V0;
200 dma_attr->dma_attr_addr_lo = 0;
201 dma_attr->dma_attr_addr_hi = UINT64_MAX;
202 dma_attr->dma_attr_count_max = UINT64_MAX;
203 dma_attr->dma_attr_align = 512;
204 dma_attr->dma_attr_burstsizes = 0xfff;
205 dma_attr->dma_attr_minxfer = 1;
206 dma_attr->dma_attr_maxxfer = UINT64_MAX;
207 dma_attr->dma_attr_seg = UINT64_MAX;
208 dma_attr->dma_attr_sgllen = 1;
209 dma_attr->dma_attr_granular = 1;
210 dma_attr->dma_attr_flags = 0;
211
212 /*
213 * Device access and DMA attributes for tx buffers
214 */
215 acc_attr = &sc->sge.acc_attr_tx;
216 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
217 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
218
219 dma_attr = &sc->sge.dma_attr_tx;
220 dma_attr->dma_attr_version = DMA_ATTR_V0;
221 dma_attr->dma_attr_addr_lo = 0;
222 dma_attr->dma_attr_addr_hi = UINT64_MAX;
223 dma_attr->dma_attr_count_max = UINT64_MAX;
224 dma_attr->dma_attr_align = 1;
225 dma_attr->dma_attr_burstsizes = 0xfff;
226 dma_attr->dma_attr_minxfer = 1;
227 dma_attr->dma_attr_maxxfer = UINT64_MAX;
228 dma_attr->dma_attr_seg = UINT64_MAX;
229 dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
230 dma_attr->dma_attr_granular = 1;
231 dma_attr->dma_attr_flags = 0;
232
233 /*
234 * Ingress Padding Boundary and Egress Status Page Size are set up by
235 * t4_fixup_host_params().
236 */
237 sge_control = t4_read_reg(sc, A_SGE_CONTROL);
238 sc->sge.pktshift = G_PKTSHIFT(sge_control);
239 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
240
241 /* t4_nex uses FLM packed mode */
242 sc->sge.fl_align = t4_fl_pkt_align(sc, true);
243
244 /*
245 * Device access and DMA attributes for rx buffers
246 */
247 sc->sge.rxb_params.dip = sc->dip;
248 sc->sge.rxb_params.buf_size = rx_buf_size;
249
250 acc_attr = &sc->sge.rxb_params.acc_attr_rx;
251 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
252 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
253
254 dma_attr = &sc->sge.rxb_params.dma_attr_rx;
255 dma_attr->dma_attr_version = DMA_ATTR_V0;
256 dma_attr->dma_attr_addr_lo = 0;
257 dma_attr->dma_attr_addr_hi = UINT64_MAX;
258 dma_attr->dma_attr_count_max = UINT64_MAX;
259 /*
260 * Low 4 bits of an rx buffer address have a special meaning to the SGE
261 * and an rx buf cannot have an address with any of these bits set.
262 * FL_ALIGN is >= 32 so we're sure things are ok.
263 */
264 dma_attr->dma_attr_align = sc->sge.fl_align;
265 dma_attr->dma_attr_burstsizes = 0xfff;
266 dma_attr->dma_attr_minxfer = 1;
267 dma_attr->dma_attr_maxxfer = UINT64_MAX;
268 dma_attr->dma_attr_seg = UINT64_MAX;
269 dma_attr->dma_attr_sgllen = 1;
270 dma_attr->dma_attr_granular = 1;
271 dma_attr->dma_attr_flags = 0;
272
273 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
274
275 /*
276 * A FL with <= fl_starve_thres buffers is starving and a periodic
277 * timer will attempt to refill it. This needs to be larger than the
278 * SGE's Egress Congestion Threshold. If it isn't, then we can get
279 * stuck waiting for new packets while the SGE is waiting for us to
280 * give it more Free List entries. (Note that the SGE's Egress
281 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
282 * there was only a single field to control this. For T5 there's the
283 * original field which now only applies to Unpacked Mode Free List
284 * buffers and a new field which only applies to Packed Mode Free List
285 * buffers.
286 */
287
288 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL);
289 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
290 case CHELSIO_T4:
291 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
292 break;
293 case CHELSIO_T5:
294 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
295 break;
296 case CHELSIO_T6:
297 default:
298 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl);
299 }
300 sc->sge.fl_starve_threshold = 2*egress_threshold + 1;
301
302 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
303
304 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
305 V_THRESHOLD_0(p->counter_val[0]) |
306 V_THRESHOLD_1(p->counter_val[1]) |
307 V_THRESHOLD_2(p->counter_val[2]) |
308 V_THRESHOLD_3(p->counter_val[3]));
309
310 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
311 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
312 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
313 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
314 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
315 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
316 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
317 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
318 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
319
320 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
321 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
322 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
323 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
324 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL,
325 t4_handle_fw_rpl);
326 }
327
328 /*
329 * Allocate and initialize the firmware event queue and the forwarded interrupt
330 * queues, if any. The adapter owns all these queues as they are not associated
331 * with any particular port.
332 *
333 * Returns errno on failure. Resources allocated up to that point may still be
334 * allocated. Caller is responsible for cleanup in case this function fails.
335 */
336 int
337 t4_setup_adapter_queues(struct adapter *sc)
338 {
339 int rc;
340
341 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
342
343 /*
344 * Firmware event queue
345 */
346 rc = alloc_fwq(sc);
347 if (rc != 0)
348 return (rc);
349
350 #ifdef TCP_OFFLOAD_ENABLE
351 /*
352 * Management queue. This is just a control queue that uses the fwq as
353 * its associated iq.
354 */
355 rc = alloc_mgmtq(sc);
356 #endif
357
358 return (rc);
359 }
360
361 /*
362 * Idempotent
363 */
364 int
365 t4_teardown_adapter_queues(struct adapter *sc)
366 {
367
368 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
369
370 (void) free_fwq(sc);
371
372 return (0);
373 }
374
375 static inline int
376 first_vector(struct port_info *pi)
377 {
378 struct adapter *sc = pi->adapter;
379 int rc = T4_EXTRA_INTR, i;
380
381 if (sc->intr_count == 1)
382 return (0);
383
384 for_each_port(sc, i) {
385 struct port_info *p = sc->port[i];
386
387 if (i == pi->port_id)
388 break;
389
390 #ifdef TCP_OFFLOAD_ENABLE
391 if (!(sc->flags & INTR_FWD))
392 rc += p->nrxq + p->nofldrxq;
393 else
394 rc += max(p->nrxq, p->nofldrxq);
395 #else
396 /*
397 * Not compiled with offload support and intr_count > 1. Only
398 * NIC queues exist and they'd better be taking direct
399 * interrupts.
400 */
401 ASSERT(!(sc->flags & INTR_FWD));
402 rc += p->nrxq;
403 #endif
404 }
405 return (rc);
406 }
407
408 /*
409 * Given an arbitrary "index," come up with an iq that can be used by other
410 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
411 * The iq returned is guaranteed to be something that takes direct interrupts.
412 */
413 static struct sge_iq *
414 port_intr_iq(struct port_info *pi, int idx)
415 {
416 struct adapter *sc = pi->adapter;
417 struct sge *s = &sc->sge;
418 struct sge_iq *iq = NULL;
419
420 if (sc->intr_count == 1)
421 return (&sc->sge.fwq);
422
423 #ifdef TCP_OFFLOAD_ENABLE
424 if (!(sc->flags & INTR_FWD)) {
425 idx %= pi->nrxq + pi->nofldrxq;
426
427 if (idx >= pi->nrxq) {
428 idx -= pi->nrxq;
429 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
430 } else
431 iq = &s->rxq[pi->first_rxq + idx].iq;
432
433 } else {
434 idx %= max(pi->nrxq, pi->nofldrxq);
435
436 if (pi->nrxq >= pi->nofldrxq)
437 iq = &s->rxq[pi->first_rxq + idx].iq;
438 else
439 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
440 }
441 #else
442 /*
443 * Not compiled with offload support and intr_count > 1. Only NIC
444 * queues exist and they'd better be taking direct interrupts.
445 */
446 ASSERT(!(sc->flags & INTR_FWD));
447
448 idx %= pi->nrxq;
449 iq = &s->rxq[pi->first_rxq + idx].iq;
450 #endif
451
452 return (iq);
453 }
454
455 int
456 t4_setup_port_queues(struct port_info *pi)
457 {
458 int rc = 0, i, intr_idx, j;
459 struct sge_rxq *rxq;
460 struct sge_txq *txq;
461 #ifdef TCP_OFFLOAD_ENABLE
462 int iqid;
463 struct sge_wrq *ctrlq;
464 struct sge_ofld_rxq *ofld_rxq;
465 struct sge_wrq *ofld_txq;
466 #endif
467 struct adapter *sc = pi->adapter;
468 struct driver_properties *p = &sc->props;
469
470 pi->ksp_config = setup_port_config_kstats(pi);
471 pi->ksp_info = setup_port_info_kstats(pi);
472
473 /* Interrupt vector to start from (when using multiple vectors) */
474 intr_idx = first_vector(pi);
475
476 /*
477 * First pass over all rx queues (NIC and TOE):
478 * a) initialize iq and fl
479 * b) allocate queue iff it will take direct interrupts.
480 */
481
482 for_each_rxq(pi, i, rxq) {
483
484 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
485 RX_IQ_ESIZE);
486
487 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
488
489 if ((!(sc->flags & INTR_FWD))
490 #ifdef TCP_OFFLOAD_ENABLE
491 || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
492 #else
493 || (sc->intr_count > 1 && pi->nrxq)
494 #endif
495 ) {
496 rxq->iq.flags |= IQ_INTR;
497 rc = alloc_rxq(pi, rxq, intr_idx, i);
498 if (rc != 0)
499 goto done;
500 intr_idx++;
501 }
502
503 }
504
505 #ifdef TCP_OFFLOAD_ENABLE
506 for_each_ofld_rxq(pi, i, ofld_rxq) {
507
508 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
509 p->qsize_rxq, RX_IQ_ESIZE);
510
511 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8);
512
513 if (!(sc->flags & INTR_FWD) ||
514 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
515 ofld_rxq->iq.flags = IQ_INTR;
516 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
517 if (rc != 0)
518 goto done;
519
520 intr_idx++;
521 }
522 }
523 #endif
524
525 /*
526 * Second pass over all rx queues (NIC and TOE). The queues forwarding
527 * their interrupts are allocated now.
528 */
529 j = 0;
530 for_each_rxq(pi, i, rxq) {
531 if (rxq->iq.flags & IQ_INTR)
532 continue;
533
534 intr_idx = port_intr_iq(pi, j)->abs_id;
535
536 rc = alloc_rxq(pi, rxq, intr_idx, i);
537 if (rc != 0)
538 goto done;
539 j++;
540 }
541
542 #ifdef TCP_OFFLOAD_ENABLE
543 for_each_ofld_rxq(pi, i, ofld_rxq) {
544 if (ofld_rxq->iq.flags & IQ_INTR)
545 continue;
546
547 intr_idx = port_intr_iq(pi, j)->abs_id;
548 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
549 if (rc != 0)
550 goto done;
551 j++;
552 }
553 #endif
554 /*
555 * Now the tx queues. Only one pass needed.
556 */
557 j = 0;
558 for_each_txq(pi, i, txq) {
559 uint16_t iqid;
560
561 iqid = port_intr_iq(pi, j)->cntxt_id;
562 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid);
563 rc = alloc_txq(pi, txq, i);
564 if (rc != 0)
565 goto done;
566 }
567
568 #ifdef TCP_OFFLOAD_ENABLE
569 for_each_ofld_txq(pi, i, ofld_txq) {
570 uint16_t iqid;
571
572 iqid = port_intr_iq(pi, j)->cntxt_id;
573 init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan,
574 iqid);
575 rc = alloc_wrq(sc, pi, ofld_txq, i);
576 if (rc != 0)
577 goto done;
578 }
579
580 /*
581 * Finally, the control queue.
582 */
583 ctrlq = &sc->sge.ctrlq[pi->port_id];
584 iqid = port_intr_iq(pi, 0)->cntxt_id;
585 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid);
586 rc = alloc_wrq(sc, pi, ctrlq, 0);
587 #endif
588
589 done:
590 if (rc != 0)
591 (void) t4_teardown_port_queues(pi);
592
593 return (rc);
594 }
595
596 /*
597 * Idempotent
598 */
599 int
600 t4_teardown_port_queues(struct port_info *pi)
601 {
602 int i;
603 struct sge_rxq *rxq;
604 struct sge_txq *txq;
605 #ifdef TCP_OFFLOAD_ENABLE
606 struct adapter *sc = pi->adapter;
607 struct sge_ofld_rxq *ofld_rxq;
608 struct sge_wrq *ofld_txq;
609 #endif
610
611 if (pi->ksp_config != NULL) {
612 kstat_delete(pi->ksp_config);
613 pi->ksp_config = NULL;
614 }
615 if (pi->ksp_info != NULL) {
616 kstat_delete(pi->ksp_info);
617 pi->ksp_info = NULL;
618 }
619
620 #ifdef TCP_OFFLOAD_ENABLE
621 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
622 #endif
623
624 for_each_txq(pi, i, txq) {
625 (void) free_txq(pi, txq);
626 }
627
628 #ifdef TCP_OFFLOAD_ENABLE
629 for_each_ofld_txq(pi, i, ofld_txq) {
630 (void) free_wrq(sc, ofld_txq);
631 }
632
633 for_each_ofld_rxq(pi, i, ofld_rxq) {
634 if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
635 (void) free_ofld_rxq(pi, ofld_rxq);
636 }
637 #endif
638
639 for_each_rxq(pi, i, rxq) {
640 if ((rxq->iq.flags & IQ_INTR) == 0)
641 (void) free_rxq(pi, rxq);
642 }
643
644 /*
645 * Then take down the rx queues that take direct interrupts.
646 */
647
648 for_each_rxq(pi, i, rxq) {
649 if (rxq->iq.flags & IQ_INTR)
650 (void) free_rxq(pi, rxq);
651 }
652
653 #ifdef TCP_OFFLOAD_ENABLE
654 for_each_ofld_rxq(pi, i, ofld_rxq) {
655 if (ofld_rxq->iq.flags & IQ_INTR)
656 (void) free_ofld_rxq(pi, ofld_rxq);
657 }
658 #endif
659
660 return (0);
661 }
662
663 /* Deals with errors and forwarded interrupts */
664 uint_t
665 t4_intr_all(caddr_t arg1, caddr_t arg2)
666 {
667
668 (void) t4_intr_err(arg1, arg2);
669 (void) t4_intr(arg1, arg2);
670
671 return (DDI_INTR_CLAIMED);
672 }
673
674 static void
675 t4_intr_rx_work(struct sge_iq *iq)
676 {
677 mblk_t *mp = NULL;
678 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
679 RXQ_LOCK(rxq);
680 if (!iq->polling) {
681 mp = t4_ring_rx(rxq, iq->qsize/8);
682 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
683 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
684 }
685 RXQ_UNLOCK(rxq);
686 if (mp != NULL)
687 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
688 rxq->ring_gen_num);
689 }
690
691 /* Deals with interrupts on the given ingress queue */
692 /* ARGSUSED */
693 uint_t
694 t4_intr(caddr_t arg1, caddr_t arg2)
695 {
696 struct sge_iq *iq = (struct sge_iq *)arg2;
697 int state;
698
699 /* Right now receive polling is only enabled for MSI-X and
700 * when we have enough msi-x vectors i.e no interrupt forwarding.
701 */
702 if (iq->adapter->props.multi_rings) {
703 t4_intr_rx_work(iq);
704 } else {
705 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
706 if (state == IQS_IDLE) {
707 (void) service_iq(iq, 0);
708 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
709 }
710 }
711 return (DDI_INTR_CLAIMED);
712 }
713
714 /* Deals with error interrupts */
715 /* ARGSUSED */
716 uint_t
717 t4_intr_err(caddr_t arg1, caddr_t arg2)
718 {
719 /* LINTED: E_BAD_PTR_CAST_ALIGN */
720 struct adapter *sc = (struct adapter *)arg1;
721
722 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
723 (void) t4_slow_intr_handler(sc);
724
725 return (DDI_INTR_CLAIMED);
726 }
727
728 /*
729 * t4_ring_rx - Process responses from an SGE response queue.
730 *
731 * This function processes responses from an SGE response queue up to the supplied budget.
732 * Responses include received packets as well as control messages from FW
733 * or HW.
734 * It returns a chain of mblks containing the received data, to be
735 * passed up to mac_ring_rx().
736 */
737 mblk_t *
738 t4_ring_rx(struct sge_rxq *rxq, int budget)
739 {
740 struct sge_iq *iq = &rxq->iq;
741 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
742 struct adapter *sc = iq->adapter;
743 struct rsp_ctrl *ctrl;
744 const struct rss_header *rss;
745 int ndescs = 0, fl_bufs_used = 0;
746 int rsp_type;
747 uint32_t lq;
748 mblk_t *mblk_head = NULL, **mblk_tail, *m;
749 struct cpl_rx_pkt *cpl;
750 uint32_t received_bytes = 0, pkt_len = 0;
751 bool csum_ok;
752 uint16_t err_vec;
753
754 mblk_tail = &mblk_head;
755
756 while (is_new_response(iq, &ctrl)) {
757
758 membar_consumer();
759
760 m = NULL;
761 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
762 lq = be32_to_cpu(ctrl->pldbuflen_qid);
763 rss = (const void *)iq->cdesc;
764
765 switch (rsp_type) {
766 case X_RSPD_TYPE_FLBUF:
767
768 ASSERT(iq->flags & IQ_HAS_FL);
769
770 if (CPL_RX_PKT == rss->opcode) {
771 cpl = (void *)(rss + 1);
772 pkt_len = be16_to_cpu(cpl->len);
773
774 if (iq->polling && ((received_bytes + pkt_len) > budget))
775 goto done;
776
777 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
778 if (m == NULL) {
779 panic("%s: line %d.", __func__,
780 __LINE__);
781 }
782
783 iq->intr_next = iq->intr_params;
784 m->b_rptr += sc->sge.pktshift;
785 if (sc->params.tp.rx_pkt_encap)
786 /* It is enabled only in T6 config file */
787 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
788 else
789 err_vec = ntohs(cpl->err_vec);
790
791 csum_ok = cpl->csum_calc && !err_vec;
792
793 /* TODO: what about cpl->ip_frag? */
794 if (csum_ok && !cpl->ip_frag) {
795 mac_hcksum_set(m, 0, 0, 0, 0xffff,
796 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
797 HCK_IPV4_HDRCKSUM_OK);
798 rxq->rxcsum++;
799 }
800 rxq->rxpkts++;
801 rxq->rxbytes += pkt_len;
802 received_bytes += pkt_len;
803
804 *mblk_tail = m;
805 mblk_tail = &m->b_next;
806
807 break;
808 }
809
810 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
811 if (m == NULL) {
812 panic("%s: line %d.", __func__,
813 __LINE__);
814 }
815
816 case X_RSPD_TYPE_CPL:
817 ASSERT(rss->opcode < NUM_CPL_CMDS);
818 sc->cpl_handler[rss->opcode](iq, rss, m);
819 break;
820
821 default:
822 break;
823 }
824 iq_next(iq);
825 ++ndescs;
826 if (!iq->polling && (ndescs == budget))
827 break;
828 }
829
830 done:
831
832 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
833 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
834 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
835
836 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) {
837 int starved;
838 FL_LOCK(fl);
839 fl->needed += fl_bufs_used;
840 starved = refill_fl(sc, fl, fl->cap / 8);
841 FL_UNLOCK(fl);
842 if (starved)
843 add_fl_to_sfl(sc, fl);
844 }
845 return (mblk_head);
846 }
847
848 /*
849 * Deals with anything and everything on the given ingress queue.
850 */
851 static int
852 service_iq(struct sge_iq *iq, int budget)
853 {
854 struct sge_iq *q;
855 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
856 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
857 struct adapter *sc = iq->adapter;
858 struct rsp_ctrl *ctrl;
859 const struct rss_header *rss;
860 int ndescs = 0, limit, fl_bufs_used = 0;
861 int rsp_type;
862 uint32_t lq;
863 mblk_t *m;
864 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
865
866 limit = budget ? budget : iq->qsize / 8;
867
868 /*
869 * We always come back and check the descriptor ring for new indirect
870 * interrupts and other responses after running a single handler.
871 */
872 for (;;) {
873 while (is_new_response(iq, &ctrl)) {
874
875 membar_consumer();
876
877 m = NULL;
878 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
879 lq = be32_to_cpu(ctrl->pldbuflen_qid);
880 rss = (const void *)iq->cdesc;
881
882 switch (rsp_type) {
883 case X_RSPD_TYPE_FLBUF:
884
885 ASSERT(iq->flags & IQ_HAS_FL);
886
887 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
888 if (m == NULL) {
889 panic("%s: line %d.", __func__,
890 __LINE__);
891 }
892
893 /* FALLTHRU */
894 case X_RSPD_TYPE_CPL:
895
896 ASSERT(rss->opcode < NUM_CPL_CMDS);
897 sc->cpl_handler[rss->opcode](iq, rss, m);
898 break;
899
900 case X_RSPD_TYPE_INTR:
901
902 /*
903 * Interrupts should be forwarded only to queues
904 * that are not forwarding their interrupts.
905 * This means service_iq can recurse but only 1
906 * level deep.
907 */
908 ASSERT(budget == 0);
909
910 q = sc->sge.iqmap[lq - sc->sge.iq_start];
911 if (atomic_cas_uint(&q->state, IQS_IDLE,
912 IQS_BUSY) == IQS_IDLE) {
913 if (service_iq(q, q->qsize / 8) == 0) {
914 (void) atomic_cas_uint(
915 &q->state, IQS_BUSY,
916 IQS_IDLE);
917 } else {
918 STAILQ_INSERT_TAIL(&iql, q,
919 link);
920 }
921 }
922 break;
923
924 default:
925 break;
926 }
927
928 iq_next(iq);
929 if (++ndescs == limit) {
930 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
931 V_CIDXINC(ndescs) |
932 V_INGRESSQID(iq->cntxt_id) |
933 V_SEINTARM(V_QINTR_TIMER_IDX(
934 X_TIMERREG_UPDATE_CIDX)));
935 ndescs = 0;
936
937 if (fl_bufs_used > 0) {
938 ASSERT(iq->flags & IQ_HAS_FL);
939 FL_LOCK(fl);
940 fl->needed += fl_bufs_used;
941 (void) refill_fl(sc, fl, fl->cap / 8);
942 FL_UNLOCK(fl);
943 fl_bufs_used = 0;
944 }
945
946 if (budget != 0)
947 return (EINPROGRESS);
948 }
949 }
950
951 if (STAILQ_EMPTY(&iql) != 0)
952 break;
953
954 /*
955 * Process the head only, and send it to the back of the list if
956 * it's still not done.
957 */
958 q = STAILQ_FIRST(&iql);
959 STAILQ_REMOVE_HEAD(&iql, link);
960 if (service_iq(q, q->qsize / 8) == 0)
961 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
962 else
963 STAILQ_INSERT_TAIL(&iql, q, link);
964 }
965
966 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
967 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
968
969 if (iq->flags & IQ_HAS_FL) {
970 int starved;
971
972 FL_LOCK(fl);
973 fl->needed += fl_bufs_used;
974 starved = refill_fl(sc, fl, fl->cap / 4);
975 FL_UNLOCK(fl);
976 if (starved != 0)
977 add_fl_to_sfl(sc, fl);
978 }
979
980 return (0);
981 }
982
983 #ifdef TCP_OFFLOAD_ENABLE
984 int
985 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
986 {
987 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
988 }
989
990 /*
991 * Doesn't fail. Holds on to work requests it can't send right away.
992 */
993 int
994 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0)
995 {
996 struct sge_eq *eq = &wrq->eq;
997 struct mblk_pair *wr_list = &wrq->wr_list;
998 int can_reclaim;
999 caddr_t dst;
1000 mblk_t *wr, *next;
1001
1002 TXQ_LOCK_ASSERT_OWNED(wrq);
1003 #ifdef TCP_OFFLOAD_ENABLE
1004 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
1005 (eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1006 #else
1007 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1008 #endif
1009
1010 if (m0 != NULL) {
1011 if (wr_list->head != NULL)
1012 wr_list->tail->b_next = m0;
1013 else
1014 wr_list->head = m0;
1015 while (m0->b_next)
1016 m0 = m0->b_next;
1017 wr_list->tail = m0;
1018 }
1019
1020 can_reclaim = reclaimable(eq);
1021 eq->cidx += can_reclaim;
1022 eq->avail += can_reclaim;
1023 if (eq->cidx >= eq->cap)
1024 eq->cidx -= eq->cap;
1025
1026 for (wr = wr_list->head; wr; wr = next) {
1027 int ndesc, len = 0;
1028 mblk_t *m;
1029
1030 next = wr->b_next;
1031 wr->b_next = NULL;
1032
1033 for (m = wr; m; m = m->b_cont)
1034 len += MBLKL(m);
1035
1036 ASSERT(len > 0 && (len & 0x7) == 0);
1037 ASSERT(len <= SGE_MAX_WR_LEN);
1038
1039 ndesc = howmany(len, EQ_ESIZE);
1040 if (eq->avail < ndesc) {
1041 wr->b_next = next;
1042 wrq->no_desc++;
1043 break;
1044 }
1045
1046 dst = (void *)&eq->desc[eq->pidx];
1047 for (m = wr; m; m = m->b_cont)
1048 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
1049
1050 eq->pidx += ndesc;
1051 eq->avail -= ndesc;
1052 if (eq->pidx >= eq->cap)
1053 eq->pidx -= eq->cap;
1054
1055 eq->pending += ndesc;
1056 if (eq->pending > 16)
1057 ring_tx_db(sc, eq);
1058
1059 wrq->tx_wrs++;
1060 freemsg(wr);
1061
1062 if (eq->avail < 8) {
1063 can_reclaim = reclaimable(eq);
1064 eq->cidx += can_reclaim;
1065 eq->avail += can_reclaim;
1066 if (eq->cidx >= eq->cap)
1067 eq->cidx -= eq->cap;
1068 }
1069 }
1070
1071 if (eq->pending != 0)
1072 ring_tx_db(sc, eq);
1073
1074 if (wr == NULL)
1075 wr_list->head = wr_list->tail = NULL;
1076 else {
1077 wr_list->head = wr;
1078
1079 ASSERT(wr_list->tail->b_next == NULL);
1080 }
1081
1082 return (0);
1083 }
1084 #endif
1085
1086 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
1087 #define TXPKTS_PKT_HDR ((\
1088 sizeof (struct ulp_txpkt) + \
1089 sizeof (struct ulptx_idata) + \
1090 sizeof (struct cpl_tx_pkt_core)) / 8)
1091
1092 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
1093 #define TXPKTS_WR_HDR (\
1094 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
1095 TXPKTS_PKT_HDR)
1096
1097 /* Header of a tx WR, before SGL of first packet (in flits) */
1098 #define TXPKT_WR_HDR ((\
1099 sizeof (struct fw_eth_tx_pkt_wr) + \
1100 sizeof (struct cpl_tx_pkt_core)) / 8)
1101
1102 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
1103 #define TXPKT_LSO_WR_HDR ((\
1104 sizeof (struct fw_eth_tx_pkt_wr) + \
1105 sizeof(struct cpl_tx_pkt_lso_core) + \
1106 sizeof (struct cpl_tx_pkt_core)) / 8)
1107
1108 mblk_t *
1109 t4_eth_tx(void *arg, mblk_t *frame)
1110 {
1111 struct sge_txq *txq = (struct sge_txq *) arg;
1112 struct port_info *pi = txq->port;
1113 struct adapter *sc = pi->adapter;
1114 struct sge_eq *eq = &txq->eq;
1115 mblk_t *next_frame;
1116 int rc, coalescing;
1117 struct txpkts txpkts;
1118 struct txinfo txinfo;
1119
1120 txpkts.npkt = 0; /* indicates there's nothing in txpkts */
1121 coalescing = 0;
1122
1123 TXQ_LOCK(txq);
1124 if (eq->avail < 8)
1125 (void) reclaim_tx_descs(txq, 8);
1126 for (; frame; frame = next_frame) {
1127
1128 if (eq->avail < 8)
1129 break;
1130
1131 next_frame = frame->b_next;
1132 frame->b_next = NULL;
1133
1134 if (next_frame != NULL)
1135 coalescing = 1;
1136
1137 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
1138 if (rc != 0) {
1139 if (rc == ENOMEM) {
1140
1141 /* Short of resources, suspend tx */
1142
1143 frame->b_next = next_frame;
1144 break;
1145 }
1146
1147 /*
1148 * Unrecoverable error for this frame, throw it
1149 * away and move on to the next.
1150 */
1151
1152 freemsg(frame);
1153 continue;
1154 }
1155
1156 if (coalescing != 0 &&
1157 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
1158
1159 /* Successfully absorbed into txpkts */
1160
1161 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
1162 goto doorbell;
1163 }
1164
1165 /*
1166 * We weren't coalescing to begin with, or current frame could
1167 * not be coalesced (add_to_txpkts flushes txpkts if a frame
1168 * given to it can't be coalesced). Either way there should be
1169 * nothing in txpkts.
1170 */
1171 ASSERT(txpkts.npkt == 0);
1172
1173 /* We're sending out individual frames now */
1174 coalescing = 0;
1175
1176 if (eq->avail < 8)
1177 (void) reclaim_tx_descs(txq, 8);
1178 rc = write_txpkt_wr(pi, txq, frame, &txinfo);
1179 if (rc != 0) {
1180
1181 /* Short of hardware descriptors, suspend tx */
1182
1183 /*
1184 * This is an unlikely but expensive failure. We've
1185 * done all the hard work (DMA bindings etc.) and now we
1186 * can't send out the frame. What's worse, we have to
1187 * spend even more time freeing up everything in txinfo.
1188 */
1189 txq->qfull++;
1190 free_txinfo_resources(txq, &txinfo);
1191
1192 frame->b_next = next_frame;
1193 break;
1194 }
1195
1196 doorbell:
1197 /* Fewer and fewer doorbells as the queue fills up */
1198 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) {
1199 txq->txbytes += txinfo.len;
1200 txq->txpkts++;
1201 ring_tx_db(sc, eq);
1202 }
1203 (void) reclaim_tx_descs(txq, 32);
1204 }
1205
1206 if (txpkts.npkt > 0)
1207 write_txpkts_wr(txq, &txpkts);
1208
1209 /*
1210 * frame not NULL means there was an error but we haven't thrown it
1211 * away. This can happen when we're short of tx descriptors (qfull) or
1212 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush
1213 * and reclaim will get things going again.
1214 *
1215 * If eq->avail is already 0 we know a credit flush was requested in the
1216 * WR that reduced it to 0 so we don't need another flush (we don't have
1217 * any descriptor for a flush WR anyway, duh).
1218 */
1219 if (frame && eq->avail > 0)
1220 write_txqflush_wr(txq);
1221
1222 if (eq->pending != 0)
1223 ring_tx_db(sc, eq);
1224
1225 (void) reclaim_tx_descs(txq, eq->qsize);
1226 TXQ_UNLOCK(txq);
1227
1228 return (frame);
1229 }
1230
1231 static inline void
1232 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx,
1233 int qsize, uint8_t esize)
1234 {
1235 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1236 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1237
1238 iq->flags = 0;
1239 iq->adapter = sc;
1240 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1241 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1242 if (pktc_idx >= 0) {
1243 iq->intr_params |= F_QINTR_CNT_EN;
1244 iq->intr_pktc_idx = pktc_idx;
1245 }
1246 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1247 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1248 }
1249
1250 static inline void
1251 init_fl(struct sge_fl *fl, uint16_t qsize)
1252 {
1253
1254 fl->qsize = qsize;
1255 }
1256
1257 static inline void
1258 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1259 uint8_t tx_chan, uint16_t iqid)
1260 {
1261 struct sge *s = &sc->sge;
1262 uint32_t r;
1263
1264 ASSERT(tx_chan < NCHAN);
1265 ASSERT(eqtype <= EQ_TYPEMASK);
1266
1267 if (is_t5(sc->params.chip)) {
1268 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1269 r >>= S_QUEUESPERPAGEPF0 +
1270 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1271 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1272 }
1273
1274 eq->flags = eqtype & EQ_TYPEMASK;
1275 eq->tx_chan = tx_chan;
1276 eq->iqid = iqid;
1277 eq->qsize = qsize;
1278 }
1279
1280 /*
1281 * Allocates the ring for an ingress queue and an optional freelist. If the
1282 * freelist is specified it will be allocated and then associated with the
1283 * ingress queue.
1284 *
1285 * Returns errno on failure. Resources allocated up to that point may still be
1286 * allocated. Caller is responsible for cleanup in case this function fails.
1287 *
1288 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1289 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies
1290 * the index of the queue to which its interrupts will be forwarded.
1291 */
1292 static int
1293 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1294 int intr_idx, int cong)
1295 {
1296 int rc, i, cntxt_id;
1297 size_t len;
1298 struct fw_iq_cmd c;
1299 struct adapter *sc = iq->adapter;
1300 uint32_t v = 0;
1301
1302 len = iq->qsize * iq->esize;
1303 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1304 &iq->ba, (caddr_t *)&iq->desc);
1305 if (rc != 0)
1306 return (rc);
1307
1308 bzero(&c, sizeof (c));
1309 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1310 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1311 V_FW_IQ_CMD_VFN(0));
1312
1313 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1314 FW_LEN16(c));
1315
1316 /* Special handling for firmware event queue */
1317 if (iq == &sc->sge.fwq)
1318 v |= F_FW_IQ_CMD_IQASYNCH;
1319
1320 if (iq->flags & IQ_INTR)
1321 ASSERT(intr_idx < sc->intr_count);
1322 else
1323 v |= F_FW_IQ_CMD_IQANDST;
1324 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1325
1326 c.type_to_iqandstindex = cpu_to_be32(v |
1327 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1328 V_FW_IQ_CMD_VIID(pi->viid) |
1329 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1330 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1331 F_FW_IQ_CMD_IQGTSMODE |
1332 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1333 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1334 c.iqsize = cpu_to_be16(iq->qsize);
1335 c.iqaddr = cpu_to_be64(iq->ba);
1336 if (cong >= 0)
1337 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN);
1338
1339 if (fl != NULL) {
1340 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip);
1341
1342 mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1343 DDI_INTR_PRI(sc->intr_pri));
1344 fl->flags |= FL_MTX;
1345
1346 len = fl->qsize * RX_FL_ESIZE;
1347 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1348 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1349 if (rc != 0)
1350 return (rc);
1351
1352 /* Allocate space for one software descriptor per buffer. */
1353 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8;
1354 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1355 KM_SLEEP);
1356 fl->needed = fl->cap;
1357 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1358
1359 c.iqns_to_fl0congen |=
1360 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1361 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1362 if (cong >= 0) {
1363 c.iqns_to_fl0congen |=
1364 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1365 F_FW_IQ_CMD_FL0CONGCIF |
1366 F_FW_IQ_CMD_FL0CONGEN);
1367 }
1368
1369 /* In T6, for egress queue type FL there is internal overhead
1370 * of 16B for header going into FLM module. Hence the maximum
1371 * allowed burst size is 448 bytes. For T4/T5, the hardware
1372 * doesn't coalesce fetch requests if more than 64 bytes of
1373 * Free List pointers are provided, so we use a 128-byte Fetch
1374 * Burst Minimum there (T6 implements coalescing so we can use
1375 * the smaller 64-byte value there).
1376 */
1377
1378 c.fl0dcaen_to_fl0cidxfthresh =
1379 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5
1380 ? X_FETCHBURSTMIN_128B
1381 : X_FETCHBURSTMIN_64B) |
1382 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5
1383 ? X_FETCHBURSTMAX_512B
1384 : X_FETCHBURSTMAX_256B));
1385 c.fl0size = cpu_to_be16(fl->qsize);
1386 c.fl0addr = cpu_to_be64(fl->ba);
1387 }
1388
1389 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1390 if (rc != 0) {
1391 cxgb_printf(sc->dip, CE_WARN,
1392 "failed to create ingress queue: %d", rc);
1393 return (rc);
1394 }
1395
1396 iq->cdesc = iq->desc;
1397 iq->cidx = 0;
1398 iq->gen = 1;
1399 iq->intr_next = iq->intr_params;
1400 iq->adapter = sc;
1401 iq->cntxt_id = be16_to_cpu(c.iqid);
1402 iq->abs_id = be16_to_cpu(c.physiqid);
1403 iq->flags |= IQ_ALLOCATED;
1404 mutex_init(&iq->lock, NULL,
1405 MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri)));
1406 iq->polling = 0;
1407
1408 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1409 if (cntxt_id >= sc->sge.niq) {
1410 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1411 cntxt_id, sc->sge.niq - 1);
1412 }
1413 sc->sge.iqmap[cntxt_id] = iq;
1414
1415 if (fl != NULL) {
1416 fl->cntxt_id = be16_to_cpu(c.fl0id);
1417 fl->pidx = fl->cidx = 0;
1418 fl->copy_threshold = rx_copy_threshold;
1419
1420 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1421 if (cntxt_id >= sc->sge.neq) {
1422 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1423 __func__, cntxt_id, sc->sge.neq - 1);
1424 }
1425 sc->sge.eqmap[cntxt_id] = (void *)fl;
1426
1427 FL_LOCK(fl);
1428 (void) refill_fl(sc, fl, fl->lowat);
1429 FL_UNLOCK(fl);
1430
1431 iq->flags |= IQ_HAS_FL;
1432 }
1433
1434 if (is_t5(sc->params.chip) && cong >= 0) {
1435 uint32_t param, val;
1436
1437 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1438 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
1439 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
1440 if (cong == 0)
1441 val = 1 << 19;
1442 else {
1443 val = 2 << 19;
1444 for (i = 0; i < 4; i++) {
1445 if (cong & (1 << i))
1446 val |= 1 << (i << 2);
1447 }
1448 }
1449
1450 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1451 if (rc != 0) {
1452 /* report error but carry on */
1453 cxgb_printf(sc->dip, CE_WARN,
1454 "failed to set congestion manager context for "
1455 "ingress queue %d: %d", iq->cntxt_id, rc);
1456 }
1457 }
1458
1459 /* Enable IQ interrupts */
1460 iq->state = IQS_IDLE;
1461 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1462 V_INGRESSQID(iq->cntxt_id));
1463
1464 return (0);
1465 }
1466
1467 static int
1468 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1469 {
1470 int rc;
1471 struct adapter *sc = iq->adapter;
1472 dev_info_t *dip;
1473
1474 dip = pi ? pi->dip : sc->dip;
1475
1476 if (iq != NULL) {
1477 if (iq->flags & IQ_ALLOCATED) {
1478 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1479 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1480 fl ? fl->cntxt_id : 0xffff, 0xffff);
1481 if (rc != 0) {
1482 cxgb_printf(dip, CE_WARN,
1483 "failed to free queue %p: %d", iq, rc);
1484 return (rc);
1485 }
1486 mutex_destroy(&iq->lock);
1487 iq->flags &= ~IQ_ALLOCATED;
1488 }
1489
1490 if (iq->desc != NULL) {
1491 (void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1492 iq->desc = NULL;
1493 }
1494
1495 bzero(iq, sizeof (*iq));
1496 }
1497
1498 if (fl != NULL) {
1499 if (fl->sdesc != NULL) {
1500 FL_LOCK(fl);
1501 free_fl_bufs(fl);
1502 FL_UNLOCK(fl);
1503
1504 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1505 fl->cap);
1506 fl->sdesc = NULL;
1507 }
1508
1509 if (fl->desc != NULL) {
1510 (void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1511 fl->desc = NULL;
1512 }
1513
1514 if (fl->flags & FL_MTX) {
1515 mutex_destroy(&fl->lock);
1516 fl->flags &= ~FL_MTX;
1517 }
1518
1519 bzero(fl, sizeof (struct sge_fl));
1520 }
1521
1522 return (0);
1523 }
1524
1525 static int
1526 alloc_fwq(struct adapter *sc)
1527 {
1528 int rc, intr_idx;
1529 struct sge_iq *fwq = &sc->sge.fwq;
1530
1531 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1532 fwq->flags |= IQ_INTR; /* always */
1533 intr_idx = sc->intr_count > 1 ? 1 : 0;
1534 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1535 if (rc != 0) {
1536 cxgb_printf(sc->dip, CE_WARN,
1537 "failed to create firmware event queue: %d.", rc);
1538 return (rc);
1539 }
1540
1541 return (0);
1542 }
1543
1544 static int
1545 free_fwq(struct adapter *sc)
1546 {
1547
1548 return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1549 }
1550
1551 #ifdef TCP_OFFLOAD_ENABLE
1552 static int
1553 alloc_mgmtq(struct adapter *sc)
1554 {
1555 int rc;
1556 struct sge_wrq *mgmtq = &sc->sge.mgmtq;
1557
1558 init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
1559 sc->sge.fwq.cntxt_id);
1560 rc = alloc_wrq(sc, NULL, mgmtq, 0);
1561 if (rc != 0) {
1562 cxgb_printf(sc->dip, CE_WARN,
1563 "failed to create management queue: %d\n", rc);
1564 return (rc);
1565 }
1566
1567 return (0);
1568 }
1569 #endif
1570
1571 static int
1572 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1573 {
1574 int rc;
1575
1576 rxq->port = pi;
1577 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1 << pi->tx_chan);
1578 if (rc != 0)
1579 return (rc);
1580
1581 rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1582
1583 return (rc);
1584 }
1585
1586 static int
1587 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1588 {
1589 int rc;
1590
1591 if (rxq->ksp != NULL) {
1592 kstat_delete(rxq->ksp);
1593 rxq->ksp = NULL;
1594 }
1595
1596 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1597 if (rc == 0)
1598 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1599
1600 return (rc);
1601 }
1602
1603 #ifdef TCP_OFFLOAD_ENABLE
1604 static int
1605 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
1606 int intr_idx)
1607 {
1608 int rc;
1609
1610 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
1611 1 << pi->tx_chan);
1612 if (rc != 0)
1613 return (rc);
1614
1615 return (rc);
1616 }
1617
1618 static int
1619 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
1620 {
1621 int rc;
1622
1623 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
1624 if (rc == 0)
1625 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) -
1626 offsetof(struct sge_ofld_rxq, fl));
1627
1628 return (rc);
1629 }
1630 #endif
1631
1632 static int
1633 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
1634 {
1635 int rc, cntxt_id;
1636 struct fw_eq_ctrl_cmd c;
1637
1638 bzero(&c, sizeof (c));
1639
1640 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
1641 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
1642 V_FW_EQ_CTRL_CMD_VFN(0));
1643 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC |
1644 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
1645 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */
1646 c.physeqid_pkd = BE_32(0);
1647 c.fetchszm_to_iqid =
1648 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1649 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
1650 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
1651 c.dcaen_to_eqsize =
1652 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1653 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1654 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1655 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
1656 c.eqaddr = BE_64(eq->ba);
1657
1658 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1659 if (rc != 0) {
1660 cxgb_printf(sc->dip, CE_WARN,
1661 "failed to create control queue %d: %d", eq->tx_chan, rc);
1662 return (rc);
1663 }
1664 eq->flags |= EQ_ALLOCATED;
1665
1666 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid));
1667 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1668 if (cntxt_id >= sc->sge.neq)
1669 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1670 cntxt_id, sc->sge.neq - 1);
1671 sc->sge.eqmap[cntxt_id] = eq;
1672
1673 return (rc);
1674 }
1675
1676 static int
1677 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1678 {
1679 int rc, cntxt_id;
1680 struct fw_eq_eth_cmd c;
1681
1682 bzero(&c, sizeof (c));
1683
1684 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1685 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1686 V_FW_EQ_ETH_CMD_VFN(0));
1687 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1688 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1689 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1690 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1691 c.fetchszm_to_iqid =
1692 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1693 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1694 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1695 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1696 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1697 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1698 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1699 c.eqaddr = BE_64(eq->ba);
1700
1701 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1702 if (rc != 0) {
1703 cxgb_printf(pi->dip, CE_WARN,
1704 "failed to create Ethernet egress queue: %d", rc);
1705 return (rc);
1706 }
1707 eq->flags |= EQ_ALLOCATED;
1708
1709 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1710 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1711 if (cntxt_id >= sc->sge.neq)
1712 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1713 cntxt_id, sc->sge.neq - 1);
1714 sc->sge.eqmap[cntxt_id] = eq;
1715
1716 return (rc);
1717 }
1718
1719 #ifdef TCP_OFFLOAD_ENABLE
1720 static int
1721 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1722 {
1723 int rc, cntxt_id;
1724 struct fw_eq_ofld_cmd c;
1725
1726 bzero(&c, sizeof (c));
1727
1728 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
1729 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
1730 V_FW_EQ_OFLD_CMD_VFN(0));
1731 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
1732 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
1733 c.fetchszm_to_iqid =
1734 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1735 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
1736 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
1737 c.dcaen_to_eqsize =
1738 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1739 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1740 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1741 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
1742 c.eqaddr = BE_64(eq->ba);
1743
1744 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1745 if (rc != 0) {
1746 cxgb_printf(pi->dip, CE_WARN,
1747 "failed to create egress queue for TCP offload: %d", rc);
1748 return (rc);
1749 }
1750 eq->flags |= EQ_ALLOCATED;
1751
1752 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd));
1753 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1754 if (cntxt_id >= sc->sge.neq)
1755 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1756 cntxt_id, sc->sge.neq - 1);
1757 sc->sge.eqmap[cntxt_id] = eq;
1758
1759 return (rc);
1760 }
1761 #endif
1762
1763 static int
1764 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1765 {
1766 int rc;
1767 size_t len;
1768
1769 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1770 eq->flags |= EQ_MTX;
1771
1772 len = eq->qsize * EQ_ESIZE;
1773 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1774 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1775 if (rc != 0)
1776 return (rc);
1777
1778 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE;
1779 eq->spg = (void *)&eq->desc[eq->cap];
1780 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
1781 eq->pidx = eq->cidx = 0;
1782 eq->doorbells = sc->doorbells;
1783
1784 switch (eq->flags & EQ_TYPEMASK) {
1785 case EQ_CTRL:
1786 rc = ctrl_eq_alloc(sc, eq);
1787 break;
1788
1789 case EQ_ETH:
1790 rc = eth_eq_alloc(sc, pi, eq);
1791 break;
1792
1793 #ifdef TCP_OFFLOAD_ENABLE
1794 case EQ_OFLD:
1795 rc = ofld_eq_alloc(sc, pi, eq);
1796 break;
1797 #endif
1798
1799 default:
1800 panic("%s: invalid eq type %d.", __func__,
1801 eq->flags & EQ_TYPEMASK);
1802 }
1803
1804 if (eq->doorbells &
1805 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) {
1806 uint32_t s_qpp = sc->sge.s_qpp;
1807 uint32_t mask = (1 << s_qpp) - 1;
1808 volatile uint8_t *udb;
1809
1810 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET;
1811 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
1812 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
1813 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
1814 eq->doorbells &= ~DOORBELL_WCWR;
1815 else {
1816 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
1817 eq->udb_qid = 0;
1818 }
1819 eq->udb = (volatile void *)udb;
1820 }
1821
1822 if (rc != 0) {
1823 cxgb_printf(sc->dip, CE_WARN,
1824 "failed to allocate egress queue(%d): %d",
1825 eq->flags & EQ_TYPEMASK, rc);
1826 }
1827
1828 return (rc);
1829 }
1830
1831 static int
1832 free_eq(struct adapter *sc, struct sge_eq *eq)
1833 {
1834 int rc;
1835
1836 if (eq->flags & EQ_ALLOCATED) {
1837 switch (eq->flags & EQ_TYPEMASK) {
1838 case EQ_CTRL:
1839 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
1840 eq->cntxt_id);
1841 break;
1842
1843 case EQ_ETH:
1844 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
1845 eq->cntxt_id);
1846 break;
1847
1848 #ifdef TCP_OFFLOAD_ENABLE
1849 case EQ_OFLD:
1850 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
1851 eq->cntxt_id);
1852 break;
1853 #endif
1854
1855 default:
1856 panic("%s: invalid eq type %d.", __func__,
1857 eq->flags & EQ_TYPEMASK);
1858 }
1859 if (rc != 0) {
1860 cxgb_printf(sc->dip, CE_WARN,
1861 "failed to free egress queue (%d): %d",
1862 eq->flags & EQ_TYPEMASK, rc);
1863 return (rc);
1864 }
1865 eq->flags &= ~EQ_ALLOCATED;
1866 }
1867
1868 if (eq->desc != NULL) {
1869 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1870 eq->desc = NULL;
1871 }
1872
1873 if (eq->flags & EQ_MTX)
1874 mutex_destroy(&eq->lock);
1875
1876 bzero(eq, sizeof (*eq));
1877 return (0);
1878 }
1879
1880 #ifdef TCP_OFFLOAD_ENABLE
1881 /* ARGSUSED */
1882 static int
1883 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
1884 int idx)
1885 {
1886 int rc;
1887
1888 rc = alloc_eq(sc, pi, &wrq->eq);
1889 if (rc != 0)
1890 return (rc);
1891
1892 wrq->adapter = sc;
1893 wrq->wr_list.head = NULL;
1894 wrq->wr_list.tail = NULL;
1895
1896 /*
1897 * TODO: use idx to figure out what kind of wrq this is and install
1898 * useful kstats for it.
1899 */
1900
1901 return (rc);
1902 }
1903
1904 static int
1905 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
1906 {
1907 int rc;
1908
1909 rc = free_eq(sc, &wrq->eq);
1910 if (rc != 0)
1911 return (rc);
1912
1913 bzero(wrq, sizeof (*wrq));
1914 return (0);
1915 }
1916 #endif
1917
1918 static int
1919 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1920 {
1921 int rc, i;
1922 struct adapter *sc = pi->adapter;
1923 struct sge_eq *eq = &txq->eq;
1924
1925 rc = alloc_eq(sc, pi, eq);
1926 if (rc != 0)
1927 return (rc);
1928
1929 txq->port = pi;
1930 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1931 txq->txb_size = eq->qsize * tx_copy_threshold;
1932 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1933 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1934 if (rc == 0)
1935 txq->txb_avail = txq->txb_size;
1936 else
1937 txq->txb_avail = txq->txb_size = 0;
1938
1939 /*
1940 * TODO: is this too low? Worst case would need around 4 times qsize
1941 * (all tx descriptors filled to the brim with SGLs, with each entry in
1942 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total
1943 * if you see too many dma_hdl_failed.
1944 */
1945 txq->tx_dhdl_total = eq->qsize * 2;
1946 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1947 txq->tx_dhdl_total, KM_SLEEP);
1948 for (i = 0; i < txq->tx_dhdl_total; i++) {
1949 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1950 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1951 if (rc != DDI_SUCCESS) {
1952 cxgb_printf(sc->dip, CE_WARN,
1953 "%s: failed to allocate DMA handle (%d)",
1954 __func__, rc);
1955 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1956 }
1957 txq->tx_dhdl_avail++;
1958 }
1959
1960 txq->ksp = setup_txq_kstats(pi, txq, idx);
1961
1962 return (rc);
1963 }
1964
1965 static int
1966 free_txq(struct port_info *pi, struct sge_txq *txq)
1967 {
1968 int i;
1969 struct adapter *sc = pi->adapter;
1970 struct sge_eq *eq = &txq->eq;
1971
1972 if (txq->ksp != NULL) {
1973 kstat_delete(txq->ksp);
1974 txq->ksp = NULL;
1975 }
1976
1977 if (txq->txb_va != NULL) {
1978 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1979 txq->txb_va = NULL;
1980 }
1981
1982 if (txq->sdesc != NULL) {
1983 struct tx_sdesc *sd;
1984 ddi_dma_handle_t hdl;
1985
1986 TXQ_LOCK(txq);
1987 while (eq->cidx != eq->pidx) {
1988 sd = &txq->sdesc[eq->cidx];
1989
1990 for (i = sd->hdls_used; i; i--) {
1991 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
1992 (void) ddi_dma_unbind_handle(hdl);
1993 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
1994 txq->tx_dhdl_cidx = 0;
1995 }
1996
1997 ASSERT(sd->m);
1998 freemsgchain(sd->m);
1999
2000 eq->cidx += sd->desc_used;
2001 if (eq->cidx >= eq->cap)
2002 eq->cidx -= eq->cap;
2003
2004 txq->txb_avail += txq->txb_used;
2005 }
2006 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
2007 ASSERT(txq->txb_avail == txq->txb_size);
2008 TXQ_UNLOCK(txq);
2009
2010 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
2011 txq->sdesc = NULL;
2012 }
2013
2014 if (txq->tx_dhdl != NULL) {
2015 for (i = 0; i < txq->tx_dhdl_total; i++) {
2016 if (txq->tx_dhdl[i] != NULL)
2017 ddi_dma_free_handle(&txq->tx_dhdl[i]);
2018 }
2019 }
2020
2021 (void) free_eq(sc, &txq->eq);
2022
2023 bzero(txq, sizeof (*txq));
2024 return (0);
2025 }
2026
2027 /*
2028 * Allocates a block of contiguous memory for DMA. Can be used to allocate
2029 * memory for descriptor rings or for tx/rx copy buffers.
2030 *
2031 * Caller does not have to clean up anything if this function fails, it cleans
2032 * up after itself.
2033 *
2034 * Caller provides the following:
2035 * len length of the block of memory to allocate.
2036 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
2037 * acc_attr device access attributes for the allocation.
2038 * dma_attr DMA attributes for the allocation
2039 *
2040 * If the function is successful it fills up this information:
2041 * dma_hdl DMA handle for the allocated memory
2042 * acc_hdl access handle for the allocated memory
2043 * ba bus address of the allocated memory
2044 * va KVA of the allocated memory.
2045 */
2046 static int
2047 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
2048 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
2049 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2050 uint64_t *pba, caddr_t *pva)
2051 {
2052 int rc;
2053 ddi_dma_handle_t dhdl;
2054 ddi_acc_handle_t ahdl;
2055 ddi_dma_cookie_t cookie;
2056 uint_t ccount;
2057 caddr_t va;
2058 size_t real_len;
2059
2060 *pva = NULL;
2061
2062 /*
2063 * DMA handle.
2064 */
2065 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
2066 if (rc != DDI_SUCCESS) {
2067 cxgb_printf(sc->dip, CE_WARN,
2068 "failed to allocate DMA handle: %d", rc);
2069
2070 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
2071 }
2072
2073 /*
2074 * Memory suitable for DMA.
2075 */
2076 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
2077 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
2078 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
2079 if (rc != DDI_SUCCESS) {
2080 cxgb_printf(sc->dip, CE_WARN,
2081 "failed to allocate DMA memory: %d", rc);
2082
2083 ddi_dma_free_handle(&dhdl);
2084 return (ENOMEM);
2085 }
2086
2087 if (len != real_len) {
2088 cxgb_printf(sc->dip, CE_WARN,
2089 "%s: len (%u) != real_len (%u)\n", len, real_len);
2090 }
2091
2092 /*
2093 * DMA bindings.
2094 */
2095 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
2096 NULL, &cookie, &ccount);
2097 if (rc != DDI_DMA_MAPPED) {
2098 cxgb_printf(sc->dip, CE_WARN,
2099 "failed to map DMA memory: %d", rc);
2100
2101 ddi_dma_mem_free(&ahdl);
2102 ddi_dma_free_handle(&dhdl);
2103 return (ENOMEM);
2104 }
2105 if (ccount != 1) {
2106 cxgb_printf(sc->dip, CE_WARN,
2107 "unusable DMA mapping (%d segments)", ccount);
2108 (void) free_desc_ring(&dhdl, &ahdl);
2109 }
2110
2111 bzero(va, real_len);
2112 *dma_hdl = dhdl;
2113 *acc_hdl = ahdl;
2114 *pba = cookie.dmac_laddress;
2115 *pva = va;
2116
2117 return (0);
2118 }
2119
2120 static int
2121 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2122 {
2123 (void) ddi_dma_unbind_handle(*dhdl);
2124 ddi_dma_mem_free(ahdl);
2125 ddi_dma_free_handle(dhdl);
2126
2127 return (0);
2128 }
2129
2130 static int
2131 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
2132 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2133 uint64_t *pba, caddr_t *pva)
2134 {
2135 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
2136 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
2137
2138 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
2139 dma_attr, dma_hdl, acc_hdl, pba, pva));
2140 }
2141
2142 static int
2143 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2144 {
2145 return (free_dma_memory(dhdl, ahdl));
2146 }
2147
2148 static int
2149 alloc_tx_copybuffer(struct adapter *sc, size_t len,
2150 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2151 uint64_t *pba, caddr_t *pva)
2152 {
2153 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
2154 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
2155
2156 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
2157 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
2158 }
2159
2160 static inline bool
2161 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
2162 {
2163 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
2164 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
2165
2166 *ctrl = (void *)((uintptr_t)iq->cdesc +
2167 (iq->esize - sizeof (struct rsp_ctrl)));
2168
2169 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
2170 }
2171
2172 static inline void
2173 iq_next(struct sge_iq *iq)
2174 {
2175 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
2176 if (++iq->cidx == iq->qsize - 1) {
2177 iq->cidx = 0;
2178 iq->gen ^= 1;
2179 iq->cdesc = iq->desc;
2180 }
2181 }
2182
2183 /*
2184 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
2185 *
2186 * Returns non-zero to indicate that it should be added to the list of starving
2187 * freelists.
2188 */
2189 static int
2190 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
2191 {
2192 uint64_t *d = &fl->desc[fl->pidx];
2193 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
2194
2195 FL_LOCK_ASSERT_OWNED(fl);
2196 ASSERT(nbufs >= 0);
2197
2198 if (nbufs > fl->needed)
2199 nbufs = fl->needed;
2200
2201 while (nbufs--) {
2202 if (sd->rxb != NULL) {
2203 if (sd->rxb->ref_cnt == 1) {
2204 /*
2205 * Buffer is available for recycling. Two ways
2206 * this can happen:
2207 *
2208 * a) All the packets DMA'd into it last time
2209 * around were within the rx_copy_threshold
2210 * and no part of the buffer was ever passed
2211 * up (ref_cnt never went over 1).
2212 *
2213 * b) Packets DMA'd into the buffer were passed
2214 * up but have all been freed by the upper
2215 * layers by now (ref_cnt went over 1 but is
2216 * now back to 1).
2217 *
2218 * Either way the bus address in the descriptor
2219 * ring is already valid.
2220 */
2221 ASSERT(*d == cpu_to_be64(sd->rxb->ba));
2222 d++;
2223 goto recycled;
2224 } else {
2225 /*
2226 * Buffer still in use and we need a
2227 * replacement. But first release our reference
2228 * on the existing buffer.
2229 */
2230 rxbuf_free(sd->rxb);
2231 }
2232 }
2233
2234 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
2235 if (sd->rxb == NULL)
2236 break;
2237 *d++ = cpu_to_be64(sd->rxb->ba);
2238
2239 recycled: fl->pending++;
2240 sd++;
2241 fl->needed--;
2242 if (++fl->pidx == fl->cap) {
2243 fl->pidx = 0;
2244 sd = fl->sdesc;
2245 d = fl->desc;
2246 }
2247 }
2248
2249 if (fl->pending >= 8)
2250 ring_fl_db(sc, fl);
2251
2252 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
2253 }
2254
2255 #ifndef TAILQ_FOREACH_SAFE
2256 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
2257 for ((var) = TAILQ_FIRST((head)); \
2258 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
2259 (var) = (tvar))
2260 #endif
2261
2262 /*
2263 * Attempt to refill all starving freelists.
2264 */
2265 static void
2266 refill_sfl(void *arg)
2267 {
2268 struct adapter *sc = arg;
2269 struct sge_fl *fl, *fl_temp;
2270
2271 mutex_enter(&sc->sfl_lock);
2272 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
2273 FL_LOCK(fl);
2274 (void) refill_fl(sc, fl, 64);
2275 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
2276 TAILQ_REMOVE(&sc->sfl, fl, link);
2277 fl->flags &= ~FL_STARVING;
2278 }
2279 FL_UNLOCK(fl);
2280 }
2281
2282 if (!TAILQ_EMPTY(&sc->sfl) != 0)
2283 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000));
2284 mutex_exit(&sc->sfl_lock);
2285 }
2286
2287 static void
2288 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
2289 {
2290 mutex_enter(&sc->sfl_lock);
2291 FL_LOCK(fl);
2292 if ((fl->flags & FL_DOOMED) == 0) {
2293 if (TAILQ_EMPTY(&sc->sfl) != 0) {
2294 sc->sfl_timer = timeout(refill_sfl, sc,
2295 drv_usectohz(100000));
2296 }
2297 fl->flags |= FL_STARVING;
2298 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
2299 }
2300 FL_UNLOCK(fl);
2301 mutex_exit(&sc->sfl_lock);
2302 }
2303
2304 static void
2305 free_fl_bufs(struct sge_fl *fl)
2306 {
2307 struct fl_sdesc *sd;
2308 unsigned int i;
2309
2310 FL_LOCK_ASSERT_OWNED(fl);
2311
2312 for (i = 0; i < fl->cap; i++) {
2313 sd = &fl->sdesc[i];
2314
2315 if (sd->rxb != NULL) {
2316 rxbuf_free(sd->rxb);
2317 sd->rxb = NULL;
2318 }
2319 }
2320 }
2321
2322 /*
2323 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2324 */
2325 static mblk_t *
2326 get_fl_payload(struct adapter *sc, struct sge_fl *fl,
2327 uint32_t len_newbuf, int *fl_bufs_used)
2328 {
2329 struct mblk_pair frame = {0};
2330 struct rxbuf *rxb;
2331 mblk_t *m = NULL;
2332 uint_t nbuf = 0, len, copy, n;
2333 uint32_t cidx, offset;
2334
2335 /*
2336 * The SGE won't pack a new frame into the current buffer if the entire
2337 * payload doesn't fit in the remaining space. Move on to the next buf
2338 * in that case.
2339 */
2340 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2341 fl->offset = 0;
2342 if (++fl->cidx == fl->cap)
2343 fl->cidx = 0;
2344 nbuf++;
2345 }
2346 cidx = fl->cidx;
2347 offset = fl->offset;
2348
2349 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2350 copy = (len <= fl->copy_threshold);
2351 if (copy != 0) {
2352 frame.head = m = allocb(len, BPRI_HI);
2353 if (m == NULL)
2354 return (NULL);
2355 }
2356
2357 while (len) {
2358 rxb = fl->sdesc[cidx].rxb;
2359 n = min(len, rxb->buf_size - offset);
2360
2361 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2362 DDI_DMA_SYNC_FORKERNEL);
2363
2364 if (copy != 0)
2365 bcopy(rxb->va + offset, m->b_wptr, n);
2366 else {
2367 m = desballoc((unsigned char *)rxb->va + offset, n,
2368 BPRI_HI, &rxb->freefunc);
2369 if (m == NULL) {
2370 freemsg(frame.head);
2371 return (NULL);
2372 }
2373 atomic_inc_uint(&rxb->ref_cnt);
2374 if (frame.head != NULL)
2375 frame.tail->b_cont = m;
2376 else
2377 frame.head = m;
2378 frame.tail = m;
2379 }
2380 m->b_wptr += n;
2381 len -= n;
2382 offset += roundup(n, sc->sge.fl_align);
2383 ASSERT(offset <= rxb->buf_size);
2384 if (offset == rxb->buf_size) {
2385 offset = 0;
2386 if (++cidx == fl->cap)
2387 cidx = 0;
2388 nbuf++;
2389 }
2390 }
2391
2392 fl->cidx = cidx;
2393 fl->offset = offset;
2394 (*fl_bufs_used) += nbuf;
2395
2396 ASSERT(frame.head != NULL);
2397 return (frame.head);
2398 }
2399
2400 /*
2401 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're
2402 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2403 * of immediate data.
2404 */
2405 #define IMM_LEN ( \
2406 2 * EQ_ESIZE \
2407 - sizeof (struct fw_eth_tx_pkt_wr) \
2408 - sizeof (struct cpl_tx_pkt_core))
2409
2410 /*
2411 * Returns non-zero on failure, no need to cleanup anything in that case.
2412 *
2413 * Note 1: We always try to pull up the mblk if required and return E2BIG only
2414 * if this fails.
2415 *
2416 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
2417 * does not have the TCP header in it.
2418 */
2419 static int
2420 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2421 int sgl_only)
2422 {
2423 uint32_t flags = 0, len, n;
2424 mblk_t *m = *fp;
2425 int rc;
2426
2427 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */
2428
2429 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2430 txinfo->flags = flags;
2431
2432 mac_lso_get(m, &txinfo->mss, &flags);
2433 txinfo->flags |= flags;
2434
2435 if (flags & HW_LSO)
2436 sgl_only = 1; /* Do not allow immediate data with LSO */
2437
2438 start: txinfo->nsegs = 0;
2439 txinfo->hdls_used = 0;
2440 txinfo->txb_used = 0;
2441 txinfo->len = 0;
2442
2443 /* total length and a rough estimate of # of segments */
2444 n = 0;
2445 for (; m; m = m->b_cont) {
2446 len = MBLKL(m);
2447 n += (len / PAGE_SIZE) + 1;
2448 txinfo->len += len;
2449 }
2450 m = *fp;
2451
2452 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2453 txq->pullup_early++;
2454 m = msgpullup(*fp, -1);
2455 if (m == NULL) {
2456 txq->pullup_failed++;
2457 return (E2BIG); /* (*fp) left as it was */
2458 }
2459 freemsg(*fp);
2460 *fp = m;
2461 mac_hcksum_set(m, NULL, NULL, NULL, NULL, txinfo->flags);
2462 }
2463
2464 if (txinfo->len <= IMM_LEN && !sgl_only)
2465 return (0); /* nsegs = 0 tells caller to use imm. tx */
2466
2467 if (txinfo->len <= txq->copy_threshold &&
2468 copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2469 goto done;
2470
2471 for (; m; m = m->b_cont) {
2472
2473 len = MBLKL(m);
2474
2475 /* Use tx copy buffer if this mblk is small enough */
2476 if (len <= txq->copy_threshold &&
2477 copy_into_txb(txq, m, len, txinfo) == 0)
2478 continue;
2479
2480 /* Add DMA bindings for this mblk to the SGL */
2481 rc = add_mblk(txq, txinfo, m, len);
2482
2483 if (rc == E2BIG ||
2484 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2485
2486 txq->pullup_late++;
2487 m = msgpullup(*fp, -1);
2488 if (m != NULL) {
2489 free_txinfo_resources(txq, txinfo);
2490 freemsg(*fp);
2491 *fp = m;
2492 mac_hcksum_set(m, NULL, NULL, NULL, NULL,
2493 txinfo->flags);
2494 goto start;
2495 }
2496
2497 txq->pullup_failed++;
2498 rc = E2BIG;
2499 }
2500
2501 if (rc != 0) {
2502 free_txinfo_resources(txq, txinfo);
2503 return (rc);
2504 }
2505 }
2506
2507 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2508
2509 done:
2510
2511 /*
2512 * Store the # of flits required to hold this frame's SGL in nflits. An
2513 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2514 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
2515 * then len1 must be set to 0.
2516 */
2517 n = txinfo->nsegs - 1;
2518 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2519 if (n & 1)
2520 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2521
2522 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2523 V_ULPTX_NSGE(txinfo->nsegs));
2524
2525 return (0);
2526 }
2527
2528 static inline int
2529 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2530 {
2531 if (txq->txb_avail < len)
2532 return (0);
2533
2534 if (txq->txb_next + len <= txq->txb_size) {
2535 *waste = 0;
2536 return (1);
2537 }
2538
2539 *waste = txq->txb_size - txq->txb_next;
2540
2541 return (txq->txb_avail - *waste < len ? 0 : 1);
2542 }
2543
2544 #define TXB_CHUNK 64
2545
2546 /*
2547 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2548 * and txq to indicate resources used. Caller has to make sure that those many
2549 * bytes are available in the mblk chain (b_cont linked).
2550 */
2551 static inline int
2552 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2553 {
2554 int waste, n;
2555
2556 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */
2557
2558 if (!fits_in_txb(txq, len, &waste)) {
2559 txq->txb_full++;
2560 return (ENOMEM);
2561 }
2562
2563 if (waste != 0) {
2564 ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2565 txinfo->txb_used += waste;
2566 txq->txb_avail -= waste;
2567 txq->txb_next = 0;
2568 }
2569
2570 for (n = 0; n < len; m = m->b_cont) {
2571 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2572 n += MBLKL(m);
2573 }
2574
2575 add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2576
2577 n = roundup(len, TXB_CHUNK);
2578 txinfo->txb_used += n;
2579 txq->txb_avail -= n;
2580 txq->txb_next += n;
2581 ASSERT(txq->txb_next <= txq->txb_size);
2582 if (txq->txb_next == txq->txb_size)
2583 txq->txb_next = 0;
2584
2585 return (0);
2586 }
2587
2588 static inline void
2589 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2590 {
2591 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */
2592
2593 if (txinfo->nsegs != 0) {
2594 int idx = txinfo->nsegs - 1;
2595 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2596 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2597 } else {
2598 txinfo->sgl.len0 = cpu_to_be32(len);
2599 txinfo->sgl.addr0 = cpu_to_be64(ba);
2600 }
2601 txinfo->nsegs++;
2602 }
2603
2604 /*
2605 * This function cleans up any partially allocated resources when it fails so
2606 * there's nothing for the caller to clean up in that case.
2607 *
2608 * EIO indicates permanent failure. Caller should drop the frame containing
2609 * this mblk and continue.
2610 *
2611 * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2612 * limit. Caller should pull up the frame before trying to send it out.
2613 * (This error means our pullup_early heuristic did not work for this frame)
2614 *
2615 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2616 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to
2617 * free up resources.
2618 */
2619 static inline int
2620 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2621 {
2622 ddi_dma_handle_t dhdl;
2623 ddi_dma_cookie_t cookie;
2624 uint_t ccount = 0;
2625 int rc;
2626
2627 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */
2628
2629 if (txq->tx_dhdl_avail == 0) {
2630 txq->dma_hdl_failed++;
2631 return (ENOMEM);
2632 }
2633
2634 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2635 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2636 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2637 &ccount);
2638 if (rc != DDI_DMA_MAPPED) {
2639 txq->dma_map_failed++;
2640
2641 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2642
2643 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2644 }
2645
2646 if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2647 (void) ddi_dma_unbind_handle(dhdl);
2648 return (E2BIG);
2649 }
2650
2651 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2652 while (--ccount) {
2653 ddi_dma_nextcookie(dhdl, &cookie);
2654 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2655 }
2656
2657 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2658 txq->tx_dhdl_pidx = 0;
2659 txq->tx_dhdl_avail--;
2660 txinfo->hdls_used++;
2661
2662 return (0);
2663 }
2664
2665 /*
2666 * Releases all the txq resources used up in the specified txinfo.
2667 */
2668 static void
2669 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2670 {
2671 int n;
2672
2673 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */
2674
2675 n = txinfo->txb_used;
2676 if (n > 0) {
2677 txq->txb_avail += n;
2678 if (n <= txq->txb_next)
2679 txq->txb_next -= n;
2680 else {
2681 n -= txq->txb_next;
2682 txq->txb_next = txq->txb_size - n;
2683 }
2684 }
2685
2686 for (n = txinfo->hdls_used; n > 0; n--) {
2687 if (txq->tx_dhdl_pidx > 0)
2688 txq->tx_dhdl_pidx--;
2689 else
2690 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2691 txq->tx_dhdl_avail++;
2692 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2693 }
2694 }
2695
2696 /*
2697 * Returns 0 to indicate that m has been accepted into a coalesced tx work
2698 * request. It has either been folded into txpkts or txpkts was flushed and m
2699 * has started a new coalesced work request (as the first frame in a fresh
2700 * txpkts).
2701 *
2702 * Returns non-zero to indicate a failure - caller is responsible for
2703 * transmitting m, if there was anything in txpkts it has been flushed.
2704 */
2705 static int
2706 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2707 struct txinfo *txinfo)
2708 {
2709 struct sge_eq *eq = &txq->eq;
2710 int can_coalesce;
2711 struct tx_sdesc *txsd;
2712 uint8_t flits;
2713
2714 TXQ_LOCK_ASSERT_OWNED(txq);
2715
2716 if (txpkts->npkt > 0) {
2717 flits = TXPKTS_PKT_HDR + txinfo->nflits;
2718 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2719 txpkts->nflits + flits <= TX_WR_FLITS &&
2720 txpkts->nflits + flits <= eq->avail * 8 &&
2721 txpkts->plen + txinfo->len < 65536;
2722
2723 if (can_coalesce != 0) {
2724 txpkts->tail->b_next = m;
2725 txpkts->tail = m;
2726 txpkts->npkt++;
2727 txpkts->nflits += flits;
2728 txpkts->plen += txinfo->len;
2729
2730 txsd = &txq->sdesc[eq->pidx];
2731 txsd->txb_used += txinfo->txb_used;
2732 txsd->hdls_used += txinfo->hdls_used;
2733
2734 return (0);
2735 }
2736
2737 /*
2738 * Couldn't coalesce m into txpkts. The first order of business
2739 * is to send txpkts on its way. Then we'll revisit m.
2740 */
2741 write_txpkts_wr(txq, txpkts);
2742 }
2743
2744 /*
2745 * Check if we can start a new coalesced tx work request with m as
2746 * the first packet in it.
2747 */
2748
2749 ASSERT(txpkts->npkt == 0);
2750 ASSERT(txinfo->len < 65536);
2751
2752 flits = TXPKTS_WR_HDR + txinfo->nflits;
2753 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2754 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2755
2756 if (can_coalesce == 0)
2757 return (EINVAL);
2758
2759 /*
2760 * Start a fresh coalesced tx WR with m as the first frame in it.
2761 */
2762 txpkts->tail = m;
2763 txpkts->npkt = 1;
2764 txpkts->nflits = flits;
2765 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2766 txpkts->plen = txinfo->len;
2767
2768 txsd = &txq->sdesc[eq->pidx];
2769 txsd->m = m;
2770 txsd->txb_used = txinfo->txb_used;
2771 txsd->hdls_used = txinfo->hdls_used;
2772
2773 return (0);
2774 }
2775
2776 /*
2777 * Note that write_txpkts_wr can never run out of hardware descriptors (but
2778 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
2779 * coalescing only if sufficient hardware descriptors are available.
2780 */
2781 static void
2782 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2783 {
2784 struct sge_eq *eq = &txq->eq;
2785 struct fw_eth_tx_pkts_wr *wr;
2786 struct tx_sdesc *txsd;
2787 uint32_t ctrl;
2788 uint16_t ndesc;
2789
2790 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2791
2792 ndesc = howmany(txpkts->nflits, 8);
2793
2794 wr = (void *)&eq->desc[eq->pidx];
2795 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2796 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2797 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2798 if (eq->avail == ndesc)
2799 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2800 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2801 wr->plen = cpu_to_be16(txpkts->plen);
2802 wr->npkt = txpkts->npkt;
2803 wr->r3 = wr->type = 0;
2804
2805 /* Everything else already written */
2806
2807 txsd = &txq->sdesc[eq->pidx];
2808 txsd->desc_used = ndesc;
2809
2810 txq->txb_used += txsd->txb_used / TXB_CHUNK;
2811 txq->hdl_used += txsd->hdls_used;
2812
2813 ASSERT(eq->avail >= ndesc);
2814
2815 eq->pending += ndesc;
2816 eq->avail -= ndesc;
2817 eq->pidx += ndesc;
2818 if (eq->pidx >= eq->cap)
2819 eq->pidx -= eq->cap;
2820
2821 txq->txpkts_pkts += txpkts->npkt;
2822 txq->txpkts_wrs++;
2823 txpkts->npkt = 0; /* emptied */
2824 }
2825
2826 static int
2827 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2828 struct txinfo *txinfo)
2829 {
2830 struct sge_eq *eq = &txq->eq;
2831 struct fw_eth_tx_pkt_wr *wr;
2832 struct cpl_tx_pkt_core *cpl;
2833 uint32_t ctrl; /* used in many unrelated places */
2834 uint64_t ctrl1;
2835 int nflits, ndesc;
2836 struct tx_sdesc *txsd;
2837 caddr_t dst;
2838
2839 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2840
2841 /*
2842 * Do we have enough flits to send this frame out?
2843 */
2844 ctrl = sizeof (struct cpl_tx_pkt_core);
2845 if (txinfo->flags & HW_LSO) {
2846 nflits = TXPKT_LSO_WR_HDR;
2847 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
2848 } else
2849 nflits = TXPKT_WR_HDR;
2850 if (txinfo->nsegs > 0)
2851 nflits += txinfo->nflits;
2852 else {
2853 nflits += howmany(txinfo->len, 8);
2854 ctrl += txinfo->len;
2855 }
2856 ndesc = howmany(nflits, 8);
2857 if (ndesc > eq->avail)
2858 return (ENOMEM);
2859
2860 /* Firmware work request header */
2861 wr = (void *)&eq->desc[eq->pidx];
2862 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2863 V_FW_WR_IMMDLEN(ctrl));
2864 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2865 if (eq->avail == ndesc)
2866 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2867 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2868 wr->r3 = 0;
2869
2870 if (txinfo->flags & HW_LSO) {
2871 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
2872 char *p = (void *)m->b_rptr;
2873 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2874 F_LSO_LAST_SLICE;
2875
2876 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2877 if (((struct ether_header *)p)->ether_type ==
2878 htons(ETHERTYPE_VLAN)) {
2879 ctrl |= V_LSO_ETHHDR_LEN(1);
2880 p += sizeof (struct ether_vlan_header);
2881 } else
2882 p += sizeof (struct ether_header);
2883
2884 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2885 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4);
2886 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2887 p += IPH_HDR_LENGTH(p);
2888 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4);
2889
2890 lso->lso_ctrl = cpu_to_be32(ctrl);
2891 lso->ipid_ofst = cpu_to_be16(0);
2892 lso->mss = cpu_to_be16(txinfo->mss);
2893 lso->seqno_offset = cpu_to_be32(0);
2894 if (is_t4(pi->adapter->params.chip))
2895 lso->len = cpu_to_be32(txinfo->len);
2896 else
2897 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len));
2898
2899 cpl = (void *)(lso + 1);
2900
2901 txq->tso_wrs++;
2902 } else
2903 cpl = (void *)(wr + 1);
2904
2905 /* Checksum offload */
2906 ctrl1 = 0;
2907 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2908 ctrl1 |= F_TXPKT_IPCSUM_DIS;
2909 if (!(txinfo->flags & HCK_FULLCKSUM))
2910 ctrl1 |= F_TXPKT_L4CSUM_DIS;
2911 if (ctrl1 == 0)
2912 txq->txcsum++; /* some hardware assistance provided */
2913
2914 /* CPL header */
2915 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2916 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2917 cpl->pack = 0;
2918 cpl->len = cpu_to_be16(txinfo->len);
2919 cpl->ctrl1 = cpu_to_be64(ctrl1);
2920
2921 /* Software descriptor */
2922 txsd = &txq->sdesc[eq->pidx];
2923 txsd->m = m;
2924 txsd->txb_used = txinfo->txb_used;
2925 txsd->hdls_used = txinfo->hdls_used;
2926 /* LINTED: E_ASSIGN_NARROW_CONV */
2927 txsd->desc_used = ndesc;
2928
2929 txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2930 txq->hdl_used += txinfo->hdls_used;
2931
2932 eq->pending += ndesc;
2933 eq->avail -= ndesc;
2934 eq->pidx += ndesc;
2935 if (eq->pidx >= eq->cap)
2936 eq->pidx -= eq->cap;
2937
2938 /* SGL */
2939 dst = (void *)(cpl + 1);
2940 if (txinfo->nsegs > 0) {
2941 txq->sgl_wrs++;
2942 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2943
2944 /* Need to zero-pad to a 16 byte boundary if not on one */
2945 if ((uintptr_t)dst & 0xf)
2946 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2947 *(uint64_t *)dst = 0;
2948
2949 } else {
2950 txq->imm_wrs++;
2951 #ifdef DEBUG
2952 ctrl = txinfo->len;
2953 #endif
2954 for (; m; m = m->b_cont) {
2955 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
2956 #ifdef DEBUG
2957 ctrl -= MBLKL(m);
2958 #endif
2959 }
2960 ASSERT(ctrl == 0);
2961 }
2962
2963 txq->txpkt_wrs++;
2964 return (0);
2965 }
2966
2967 static inline void
2968 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
2969 struct txpkts *txpkts, struct txinfo *txinfo)
2970 {
2971 struct ulp_txpkt *ulpmc;
2972 struct ulptx_idata *ulpsc;
2973 struct cpl_tx_pkt_core *cpl;
2974 uintptr_t flitp, start, end;
2975 uint64_t ctrl;
2976 caddr_t dst;
2977
2978 ASSERT(txpkts->npkt > 0);
2979
2980 start = (uintptr_t)txq->eq.desc;
2981 end = (uintptr_t)txq->eq.spg;
2982
2983 /* Checksum offload */
2984 ctrl = 0;
2985 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2986 ctrl |= F_TXPKT_IPCSUM_DIS;
2987 if (!(txinfo->flags & HCK_FULLCKSUM))
2988 ctrl |= F_TXPKT_L4CSUM_DIS;
2989 if (ctrl == 0)
2990 txq->txcsum++; /* some hardware assistance provided */
2991
2992 /*
2993 * The previous packet's SGL must have ended at a 16 byte boundary (this
2994 * is required by the firmware/hardware). It follows that flitp cannot
2995 * wrap around between the ULPTX master command and ULPTX subcommand (8
2996 * bytes each), and that it can not wrap around in the middle of the
2997 * cpl_tx_pkt_core either.
2998 */
2999 flitp = (uintptr_t)txpkts->flitp;
3000 ASSERT((flitp & 0xf) == 0);
3001
3002 /* ULP master command */
3003 ulpmc = (void *)flitp;
3004 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
3005 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
3006 sizeof (*cpl) + 8 * txinfo->nflits, 16));
3007
3008 /* ULP subcommand */
3009 ulpsc = (void *)(ulpmc + 1);
3010 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
3011 F_ULP_TX_SC_MORE);
3012 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
3013
3014 flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
3015 if (flitp == end)
3016 flitp = start;
3017
3018 /* CPL_TX_PKT */
3019 cpl = (void *)flitp;
3020 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3021 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
3022 cpl->pack = 0;
3023 cpl->len = cpu_to_be16(txinfo->len);
3024 cpl->ctrl1 = cpu_to_be64(ctrl);
3025
3026 flitp += sizeof (*cpl);
3027 if (flitp == end)
3028 flitp = start;
3029
3030 /* SGL for this frame */
3031 dst = (caddr_t)flitp;
3032 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
3033 flitp = (uintptr_t)dst;
3034
3035 /* Zero pad and advance to a 16 byte boundary if not already at one. */
3036 if (flitp & 0xf) {
3037
3038 /* no matter what, flitp should be on an 8 byte boundary */
3039 ASSERT((flitp & 0x7) == 0);
3040
3041 *(uint64_t *)flitp = 0;
3042 flitp += sizeof (uint64_t);
3043 txpkts->nflits++;
3044 }
3045
3046 if (flitp == end)
3047 flitp = start;
3048
3049 txpkts->flitp = (void *)flitp;
3050 }
3051
3052 static inline void
3053 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
3054 {
3055 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
3056 bcopy(from, *to, len);
3057 (*to) += len;
3058 } else {
3059 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
3060
3061 bcopy(from, *to, portion);
3062 from += portion;
3063 portion = len - portion; /* remaining */
3064 bcopy(from, (void *)eq->desc, portion);
3065 (*to) = (caddr_t)eq->desc + portion;
3066 }
3067 }
3068
3069 static inline void
3070 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
3071 {
3072 int val, db_mode;
3073 u_int db = eq->doorbells;
3074
3075 if (eq->pending > 1)
3076 db &= ~DOORBELL_WCWR;
3077
3078 if (eq->pending > eq->pidx) {
3079 int offset = eq->cap - (eq->pending - eq->pidx);
3080
3081 /* pidx has wrapped around since last doorbell */
3082
3083 (void) ddi_dma_sync(eq->desc_dhdl,
3084 offset * sizeof (struct tx_desc), 0,
3085 DDI_DMA_SYNC_FORDEV);
3086 (void) ddi_dma_sync(eq->desc_dhdl,
3087 0, eq->pidx * sizeof (struct tx_desc),
3088 DDI_DMA_SYNC_FORDEV);
3089 } else if (eq->pending > 0) {
3090 (void) ddi_dma_sync(eq->desc_dhdl,
3091 (eq->pidx - eq->pending) * sizeof (struct tx_desc),
3092 eq->pending * sizeof (struct tx_desc),
3093 DDI_DMA_SYNC_FORDEV);
3094 }
3095
3096 membar_producer();
3097
3098 if (is_t4(sc->params.chip))
3099 val = V_PIDX(eq->pending);
3100 else
3101 val = V_PIDX_T5(eq->pending);
3102
3103 db_mode = (1 << (ffs(db) - 1));
3104 switch (db_mode) {
3105 case DOORBELL_UDB:
3106 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3107 break;
3108
3109 case DOORBELL_WCWR:
3110 {
3111 volatile uint64_t *dst, *src;
3112 int i;
3113 /*
3114 * Queues whose 128B doorbell segment fits in
3115 * the page do not use relative qid
3116 * (udb_qid is always 0). Only queues with
3117 * doorbell segments can do WCWR.
3118 */
3119 ASSERT(eq->udb_qid == 0 && eq->pending == 1);
3120
3121 dst = (volatile void *)((uintptr_t)eq->udb +
3122 UDBS_WR_OFFSET - UDBS_DB_OFFSET);
3123 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
3124 src = (void *)&eq->desc[i];
3125 while (src != (void *)&eq->desc[i + 1])
3126 *dst++ = *src++;
3127 membar_producer();
3128 break;
3129 }
3130
3131 case DOORBELL_UDBWC:
3132 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3133 membar_producer();
3134 break;
3135
3136 case DOORBELL_KDB:
3137 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
3138 V_QID(eq->cntxt_id) | val);
3139 break;
3140 }
3141
3142 eq->pending = 0;
3143 }
3144
3145 static int
3146 reclaim_tx_descs(struct sge_txq *txq, int howmany)
3147 {
3148 struct tx_sdesc *txsd;
3149 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
3150 struct sge_eq *eq = &txq->eq;
3151
3152 EQ_LOCK_ASSERT_OWNED(eq);
3153
3154 cidx = eq->spg->cidx; /* stable snapshot */
3155 cidx = be16_to_cpu(cidx);
3156
3157 if (cidx >= eq->cidx)
3158 can_reclaim = cidx - eq->cidx;
3159 else
3160 can_reclaim = cidx + eq->cap - eq->cidx;
3161
3162 if (can_reclaim == 0)
3163 return (0);
3164
3165 txb_freed = hdls_freed = reclaimed = 0;
3166 do {
3167 int ndesc;
3168
3169 txsd = &txq->sdesc[eq->cidx];
3170 ndesc = txsd->desc_used;
3171
3172 /* Firmware doesn't return "partial" credits. */
3173 ASSERT(can_reclaim >= ndesc);
3174
3175 /*
3176 * We always keep mblk around, even for immediate data. If mblk
3177 * is NULL, this has to be the software descriptor for a credit
3178 * flush work request.
3179 */
3180 if (txsd->m != NULL)
3181 freemsgchain(txsd->m);
3182 #ifdef DEBUG
3183 else {
3184 ASSERT(txsd->txb_used == 0);
3185 ASSERT(txsd->hdls_used == 0);
3186 ASSERT(ndesc == 1);
3187 }
3188 #endif
3189
3190 txb_freed += txsd->txb_used;
3191 hdls_freed += txsd->hdls_used;
3192 reclaimed += ndesc;
3193
3194 eq->cidx += ndesc;
3195 if (eq->cidx >= eq->cap)
3196 eq->cidx -= eq->cap;
3197
3198 can_reclaim -= ndesc;
3199
3200 } while (can_reclaim && reclaimed < howmany);
3201
3202 eq->avail += reclaimed;
3203 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */
3204
3205 txq->txb_avail += txb_freed;
3206
3207 txq->tx_dhdl_avail += hdls_freed;
3208 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
3209 for (; hdls_freed; hdls_freed--) {
3210 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
3211 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
3212 txq->tx_dhdl_cidx = 0;
3213 }
3214
3215 return (reclaimed);
3216 }
3217
3218 static void
3219 write_txqflush_wr(struct sge_txq *txq)
3220 {
3221 struct sge_eq *eq = &txq->eq;
3222 struct fw_eq_flush_wr *wr;
3223 struct tx_sdesc *txsd;
3224
3225 EQ_LOCK_ASSERT_OWNED(eq);
3226 ASSERT(eq->avail > 0);
3227
3228 wr = (void *)&eq->desc[eq->pidx];
3229 bzero(wr, sizeof (*wr));
3230 wr->opcode = FW_EQ_FLUSH_WR;
3231 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
3232 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3233
3234 txsd = &txq->sdesc[eq->pidx];
3235 txsd->m = NULL;
3236 txsd->txb_used = 0;
3237 txsd->hdls_used = 0;
3238 txsd->desc_used = 1;
3239
3240 eq->pending++;
3241 eq->avail--;
3242 if (++eq->pidx == eq->cap)
3243 eq->pidx = 0;
3244 }
3245
3246 static int
3247 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3248 {
3249 bool csum_ok;
3250 uint16_t err_vec;
3251 struct sge_rxq *rxq = (void *)iq;
3252 struct mblk_pair chain = {0};
3253 struct adapter *sc = iq->adapter;
3254 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
3255
3256 iq->intr_next = iq->intr_params;
3257
3258 m->b_rptr += sc->sge.pktshift;
3259
3260 /* Compressed error vector is enabled for T6 only */
3261 if (sc->params.tp.rx_pkt_encap)
3262 /* It is enabled only in T6 config file */
3263 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
3264 else
3265 err_vec = ntohs(cpl->err_vec);
3266
3267 csum_ok = cpl->csum_calc && !err_vec;
3268 /* TODO: what about cpl->ip_frag? */
3269 if (csum_ok && !cpl->ip_frag) {
3270 mac_hcksum_set(m, 0, 0, 0, 0xffff,
3271 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
3272 HCK_IPV4_HDRCKSUM_OK);
3273 rxq->rxcsum++;
3274 }
3275
3276 /* Add to the chain that we'll send up */
3277 if (chain.head != NULL)
3278 chain.tail->b_next = m;
3279 else
3280 chain.head = m;
3281 chain.tail = m;
3282
3283 t4_mac_rx(rxq->port, rxq, chain.head);
3284
3285 rxq->rxpkts++;
3286 rxq->rxbytes += be16_to_cpu(cpl->len);
3287 return (0);
3288 }
3289
3290 #define FL_HW_IDX(idx) ((idx) >> 3)
3291
3292 static inline void
3293 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3294 {
3295 int desc_start, desc_last, ndesc;
3296 uint32_t v = sc->params.arch.sge_fl_db ;
3297
3298 ndesc = FL_HW_IDX(fl->pending);
3299
3300 /* Hold back one credit if pidx = cidx */
3301 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3302 ndesc--;
3303
3304 /*
3305 * There are chances of ndesc modified above (to avoid pidx = cidx).
3306 * If there is nothing to post, return.
3307 */
3308 if (ndesc <= 0)
3309 return;
3310
3311 desc_last = FL_HW_IDX(fl->pidx);
3312
3313 if (fl->pidx < fl->pending) {
3314 /* There was a wrap */
3315 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
3316
3317 /* From desc_start to the end of list */
3318 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
3319 DDI_DMA_SYNC_FORDEV);
3320
3321 /* From start of list to the desc_last */
3322 if (desc_last != 0)
3323 (void) ddi_dma_sync(fl->dhdl, 0, desc_last *
3324 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3325 } else {
3326 /* There was no wrap, sync from start_desc to last_desc */
3327 desc_start = FL_HW_IDX(fl->pidx - fl->pending);
3328 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
3329 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3330 }
3331
3332 if (is_t4(sc->params.chip))
3333 v |= V_PIDX(ndesc);
3334 else
3335 v |= V_PIDX_T5(ndesc);
3336 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3337
3338 membar_producer();
3339
3340 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3341
3342 /*
3343 * Update pending count:
3344 * Deduct the number of descriptors posted
3345 */
3346 fl->pending -= ndesc * 8;
3347 }
3348
3349 static void
3350 tx_reclaim_task(void *arg)
3351 {
3352 struct sge_txq *txq = arg;
3353
3354 TXQ_LOCK(txq);
3355 reclaim_tx_descs(txq, txq->eq.qsize);
3356 TXQ_UNLOCK(txq);
3357 }
3358
3359 /* ARGSUSED */
3360 static int
3361 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3362 mblk_t *m)
3363 {
3364 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3365 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3366 struct adapter *sc = iq->adapter;
3367 struct sge *s = &sc->sge;
3368 struct sge_eq *eq;
3369 struct sge_txq *txq;
3370
3371 txq = (void *)s->eqmap[qid - s->eq_start];
3372 eq = &txq->eq;
3373 txq->qflush++;
3374 t4_mac_tx_update(txq->port, txq);
3375
3376 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3377 (void *)txq, DDI_NOSLEEP);
3378
3379 return (0);
3380 }
3381
3382 static int
3383 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3384 {
3385 struct adapter *sc = iq->adapter;
3386 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3387
3388 ASSERT(m == NULL);
3389
3390 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3391 const struct rss_header *rss2;
3392
3393 rss2 = (const struct rss_header *)&cpl->data[0];
3394 return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3395 }
3396 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3397 }
3398
3399 int
3400 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3401 int flags)
3402 {
3403 int i, rc;
3404
3405 txmaps->map_total = count;
3406 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3407
3408 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) *
3409 txmaps->map_total, flags);
3410
3411 for (i = 0; i < count; i++) {
3412 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3413 DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3414 if (rc != DDI_SUCCESS) {
3415 cxgb_printf(sc->dip, CE_WARN,
3416 "%s: failed to allocate DMA handle (%d)",
3417 __func__, rc);
3418 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3419 }
3420 txmaps->map_avail++;
3421 }
3422
3423 return (0);
3424 }
3425
3426 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3427 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3428 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
3429 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x
3430 #define KS_C_SET(x, ...) \
3431 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
3432
3433 /*
3434 * cxgbe:X:config
3435 */
3436 struct cxgbe_port_config_kstats {
3437 kstat_named_t idx;
3438 kstat_named_t nrxq;
3439 kstat_named_t ntxq;
3440 kstat_named_t first_rxq;
3441 kstat_named_t first_txq;
3442 kstat_named_t controller;
3443 kstat_named_t factory_mac_address;
3444 };
3445
3446 /*
3447 * cxgbe:X:info
3448 */
3449 struct cxgbe_port_info_kstats {
3450 kstat_named_t transceiver;
3451 kstat_named_t rx_ovflow0;
3452 kstat_named_t rx_ovflow1;
3453 kstat_named_t rx_ovflow2;
3454 kstat_named_t rx_ovflow3;
3455 kstat_named_t rx_trunc0;
3456 kstat_named_t rx_trunc1;
3457 kstat_named_t rx_trunc2;
3458 kstat_named_t rx_trunc3;
3459 kstat_named_t tx_pause;
3460 kstat_named_t rx_pause;
3461 };
3462
3463 static kstat_t *
3464 setup_port_config_kstats(struct port_info *pi)
3465 {
3466 kstat_t *ksp;
3467 struct cxgbe_port_config_kstats *kstatp;
3468 int ndata;
3469 dev_info_t *pdip = ddi_get_parent(pi->dip);
3470 uint8_t *ma = &pi->hw_addr[0];
3471
3472 ndata = sizeof (struct cxgbe_port_config_kstats) /
3473 sizeof (kstat_named_t);
3474
3475 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3476 "net", KSTAT_TYPE_NAMED, ndata, 0);
3477 if (ksp == NULL) {
3478 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3479 return (NULL);
3480 }
3481
3482 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3483
3484 KS_UINIT(idx);
3485 KS_UINIT(nrxq);
3486 KS_UINIT(ntxq);
3487 KS_UINIT(first_rxq);
3488 KS_UINIT(first_txq);
3489 KS_CINIT(controller);
3490 KS_CINIT(factory_mac_address);
3491
3492 KS_U_SET(idx, pi->port_id);
3493 KS_U_SET(nrxq, pi->nrxq);
3494 KS_U_SET(ntxq, pi->ntxq);
3495 KS_U_SET(first_rxq, pi->first_rxq);
3496 KS_U_SET(first_txq, pi->first_txq);
3497 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3498 ddi_get_instance(pdip));
3499 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3500 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3501
3502 /* Do NOT set ksp->ks_update. These kstats do not change. */
3503
3504 /* Install the kstat */
3505 ksp->ks_private = (void *)pi;
3506 kstat_install(ksp);
3507
3508 return (ksp);
3509 }
3510
3511 static kstat_t *
3512 setup_port_info_kstats(struct port_info *pi)
3513 {
3514 kstat_t *ksp;
3515 struct cxgbe_port_info_kstats *kstatp;
3516 int ndata;
3517
3518 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3519
3520 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3521 "net", KSTAT_TYPE_NAMED, ndata, 0);
3522 if (ksp == NULL) {
3523 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3524 return (NULL);
3525 }
3526
3527 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3528
3529 KS_CINIT(transceiver);
3530 KS_UINIT(rx_ovflow0);
3531 KS_UINIT(rx_ovflow1);
3532 KS_UINIT(rx_ovflow2);
3533 KS_UINIT(rx_ovflow3);
3534 KS_UINIT(rx_trunc0);
3535 KS_UINIT(rx_trunc1);
3536 KS_UINIT(rx_trunc2);
3537 KS_UINIT(rx_trunc3);
3538 KS_UINIT(tx_pause);
3539 KS_UINIT(rx_pause);
3540
3541 /* Install the kstat */
3542 ksp->ks_update = update_port_info_kstats;
3543 ksp->ks_private = (void *)pi;
3544 kstat_install(ksp);
3545
3546 return (ksp);
3547 }
3548
3549 static int
3550 update_port_info_kstats(kstat_t *ksp, int rw)
3551 {
3552 struct cxgbe_port_info_kstats *kstatp =
3553 (struct cxgbe_port_info_kstats *)ksp->ks_data;
3554 struct port_info *pi = ksp->ks_private;
3555 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3556 "active TWINAX", "LRM" };
3557 uint32_t bgmap;
3558
3559 if (rw == KSTAT_WRITE)
3560 return (0);
3561
3562 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3563 KS_C_SET(transceiver, "unplugged");
3564 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3565 KS_C_SET(transceiver, "unknown");
3566 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3567 KS_C_SET(transceiver, "unsupported");
3568 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3569 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3570 else
3571 KS_C_SET(transceiver, "type %d", pi->mod_type);
3572
3573 #define GET_STAT(name) t4_read_reg64(pi->adapter, \
3574 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3575 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3576 A_MPS_STAT_##name##_L)
3577
3578 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3579 if (bgmap == 0)
3580 bgmap = (pi->port_id == 0) ? 0xf : 0;
3581 else if (bgmap == 1)
3582 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3583 else
3584 bgmap = 1;
3585
3586 KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3587 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3588 KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3589 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3590 KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3591 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3592 KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3593 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3594 KS_U_SET(rx_trunc0, (bgmap & 1) ?
3595 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3596 KS_U_SET(rx_trunc1, (bgmap & 2) ?
3597 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3598 KS_U_SET(rx_trunc2, (bgmap & 4) ?
3599 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3600 KS_U_SET(rx_trunc3, (bgmap & 8) ?
3601 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3602
3603 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3604 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3605
3606 return (0);
3607
3608 }
3609
3610 /*
3611 * cxgbe:X:rxqY
3612 */
3613 struct rxq_kstats {
3614 kstat_named_t rxcsum;
3615 kstat_named_t rxpkts;
3616 kstat_named_t rxbytes;
3617 kstat_named_t nomem;
3618 };
3619
3620 static kstat_t *
3621 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3622 {
3623 struct kstat *ksp;
3624 struct rxq_kstats *kstatp;
3625 int ndata;
3626 char str[16];
3627
3628 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3629 (void) snprintf(str, sizeof (str), "rxq%u", idx);
3630
3631 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3632 KSTAT_TYPE_NAMED, ndata, 0);
3633 if (ksp == NULL) {
3634 cxgb_printf(pi->dip, CE_WARN,
3635 "%s: failed to initialize rxq kstats for queue %d.",
3636 __func__, idx);
3637 return (NULL);
3638 }
3639
3640 kstatp = (struct rxq_kstats *)ksp->ks_data;
3641
3642 KS_UINIT(rxcsum);
3643 KS_UINIT(rxpkts);
3644 KS_UINIT(rxbytes);
3645 KS_UINIT(nomem);
3646
3647 ksp->ks_update = update_rxq_kstats;
3648 ksp->ks_private = (void *)rxq;
3649 kstat_install(ksp);
3650
3651 return (ksp);
3652 }
3653
3654 static int
3655 update_rxq_kstats(kstat_t *ksp, int rw)
3656 {
3657 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3658 struct sge_rxq *rxq = ksp->ks_private;
3659
3660 if (rw == KSTAT_WRITE)
3661 return (0);
3662
3663 KS_U_FROM(rxcsum, rxq);
3664 KS_U_FROM(rxpkts, rxq);
3665 KS_U_FROM(rxbytes, rxq);
3666 KS_U_FROM(nomem, rxq);
3667
3668 return (0);
3669 }
3670
3671 /*
3672 * cxgbe:X:txqY
3673 */
3674 struct txq_kstats {
3675 kstat_named_t txcsum;
3676 kstat_named_t tso_wrs;
3677 kstat_named_t imm_wrs;
3678 kstat_named_t sgl_wrs;
3679 kstat_named_t txpkt_wrs;
3680 kstat_named_t txpkts_wrs;
3681 kstat_named_t txpkts_pkts;
3682 kstat_named_t txb_used;
3683 kstat_named_t hdl_used;
3684 kstat_named_t txb_full;
3685 kstat_named_t dma_hdl_failed;
3686 kstat_named_t dma_map_failed;
3687 kstat_named_t qfull;
3688 kstat_named_t qflush;
3689 kstat_named_t pullup_early;
3690 kstat_named_t pullup_late;
3691 kstat_named_t pullup_failed;
3692 };
3693
3694 static kstat_t *
3695 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3696 {
3697 struct kstat *ksp;
3698 struct txq_kstats *kstatp;
3699 int ndata;
3700 char str[16];
3701
3702 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3703 (void) snprintf(str, sizeof (str), "txq%u", idx);
3704
3705 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3706 KSTAT_TYPE_NAMED, ndata, 0);
3707 if (ksp == NULL) {
3708 cxgb_printf(pi->dip, CE_WARN,
3709 "%s: failed to initialize txq kstats for queue %d.",
3710 __func__, idx);
3711 return (NULL);
3712 }
3713
3714 kstatp = (struct txq_kstats *)ksp->ks_data;
3715
3716 KS_UINIT(txcsum);
3717 KS_UINIT(tso_wrs);
3718 KS_UINIT(imm_wrs);
3719 KS_UINIT(sgl_wrs);
3720 KS_UINIT(txpkt_wrs);
3721 KS_UINIT(txpkts_wrs);
3722 KS_UINIT(txpkts_pkts);
3723 KS_UINIT(txb_used);
3724 KS_UINIT(hdl_used);
3725 KS_UINIT(txb_full);
3726 KS_UINIT(dma_hdl_failed);
3727 KS_UINIT(dma_map_failed);
3728 KS_UINIT(qfull);
3729 KS_UINIT(qflush);
3730 KS_UINIT(pullup_early);
3731 KS_UINIT(pullup_late);
3732 KS_UINIT(pullup_failed);
3733
3734 ksp->ks_update = update_txq_kstats;
3735 ksp->ks_private = (void *)txq;
3736 kstat_install(ksp);
3737
3738 return (ksp);
3739 }
3740
3741 static int
3742 update_txq_kstats(kstat_t *ksp, int rw)
3743 {
3744 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3745 struct sge_txq *txq = ksp->ks_private;
3746
3747 if (rw == KSTAT_WRITE)
3748 return (0);
3749
3750 KS_U_FROM(txcsum, txq);
3751 KS_U_FROM(tso_wrs, txq);
3752 KS_U_FROM(imm_wrs, txq);
3753 KS_U_FROM(sgl_wrs, txq);
3754 KS_U_FROM(txpkt_wrs, txq);
3755 KS_U_FROM(txpkts_wrs, txq);
3756 KS_U_FROM(txpkts_pkts, txq);
3757 KS_U_FROM(txb_used, txq);
3758 KS_U_FROM(hdl_used, txq);
3759 KS_U_FROM(txb_full, txq);
3760 KS_U_FROM(dma_hdl_failed, txq);
3761 KS_U_FROM(dma_map_failed, txq);
3762 KS_U_FROM(qfull, txq);
3763 KS_U_FROM(qflush, txq);
3764 KS_U_FROM(pullup_early, txq);
3765 KS_U_FROM(pullup_late, txq);
3766 KS_U_FROM(pullup_failed, txq);
3767
3768 return (0);
3769 }
Unleashed OS