4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
27 * sun4v LDC Link Layer
29 #include <sys/types.h>
31 #include <sys/errno.h>
36 #include <sys/cmn_err.h>
37 #include <sys/ksynch.h>
38 #include <sys/modctl.h>
39 #include <sys/stat.h> /* needed for S_IFBLK and S_IFCHR */
40 #include <sys/debug.h>
42 #include <sys/promif.h>
44 #include <sys/sunddi.h>
45 #include <sys/cyclic.h>
46 #include <sys/machsystm.h>
49 #include <sys/intreg.h>
50 #include <sys/machcpuvar.h>
55 #include <vm/hat_sfmmu.h>
56 #include <sys/vm_machparam.h>
57 #include <vm/seg_kmem.h>
58 #include <vm/seg_kpm.h>
60 #include <sys/ivintr.h>
61 #include <sys/hypervisor_api.h>
63 #include <sys/ldc_impl.h>
69 /* Core internal functions */
70 int i_ldc_h2v_error(int h_error
);
71 void i_ldc_reset(ldc_chan_t
*ldcp
, boolean_t force_reset
);
73 static int i_ldc_txq_reconf(ldc_chan_t
*ldcp
);
74 static int i_ldc_rxq_reconf(ldc_chan_t
*ldcp
, boolean_t force_reset
);
75 static void i_ldc_rxq_drain(ldc_chan_t
*ldcp
);
76 static void i_ldc_reset_state(ldc_chan_t
*ldcp
);
77 static void i_ldc_debug_enter(void);
79 static int i_ldc_get_tx_tail(ldc_chan_t
*ldcp
, uint64_t *tail
);
80 static void i_ldc_get_tx_head(ldc_chan_t
*ldcp
, uint64_t *head
);
81 static int i_ldc_set_tx_tail(ldc_chan_t
*ldcp
, uint64_t tail
);
82 static int i_ldc_set_rx_head(ldc_chan_t
*ldcp
, uint64_t head
);
83 static int i_ldc_send_pkt(ldc_chan_t
*ldcp
, uint8_t pkttype
, uint8_t subtype
,
86 static int i_ldc_set_rxdq_head(ldc_chan_t
*ldcp
, uint64_t head
);
87 static void i_ldc_rxdq_copy(ldc_chan_t
*ldcp
, uint64_t *head
);
88 static uint64_t i_ldc_dq_rx_get_state(ldc_chan_t
*ldcp
, uint64_t *head
,
89 uint64_t *tail
, uint64_t *link_state
);
90 static uint64_t i_ldc_hvq_rx_get_state(ldc_chan_t
*ldcp
, uint64_t *head
,
91 uint64_t *tail
, uint64_t *link_state
);
92 static int i_ldc_rx_ackpeek(ldc_chan_t
*ldcp
, uint64_t rx_head
,
94 static uint_t
i_ldc_chkq(ldc_chan_t
*ldcp
);
96 /* Interrupt handling functions */
97 static uint_t
i_ldc_tx_hdlr(caddr_t arg1
, caddr_t arg2
);
98 static uint_t
i_ldc_rx_hdlr(caddr_t arg1
, caddr_t arg2
);
99 static uint_t
i_ldc_rx_process_hvq(ldc_chan_t
*ldcp
, boolean_t
*notify_client
,
100 uint64_t *notify_event
);
101 static void i_ldc_clear_intr(ldc_chan_t
*ldcp
, cnex_intrtype_t itype
);
103 /* Read method functions */
104 static int i_ldc_read_raw(ldc_chan_t
*ldcp
, caddr_t target_bufp
, size_t *sizep
);
105 static int i_ldc_read_packet(ldc_chan_t
*ldcp
, caddr_t target_bufp
,
107 static int i_ldc_read_stream(ldc_chan_t
*ldcp
, caddr_t target_bufp
,
110 /* Write method functions */
111 static int i_ldc_write_raw(ldc_chan_t
*ldcp
, caddr_t target_bufp
,
113 static int i_ldc_write_packet(ldc_chan_t
*ldcp
, caddr_t target_bufp
,
115 static int i_ldc_write_stream(ldc_chan_t
*ldcp
, caddr_t target_bufp
,
118 /* Pkt processing internal functions */
119 static int i_ldc_check_seqid(ldc_chan_t
*ldcp
, ldc_msg_t
*ldcmsg
);
120 static int i_ldc_ctrlmsg(ldc_chan_t
*ldcp
, ldc_msg_t
*ldcmsg
);
121 static int i_ldc_process_VER(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
);
122 static int i_ldc_process_RTS(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
);
123 static int i_ldc_process_RTR(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
);
124 static int i_ldc_process_RDX(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
);
125 static int i_ldc_process_data_ACK(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
);
127 /* Imported functions */
128 extern void i_ldc_mem_set_hsvc_vers(uint64_t major
, uint64_t minor
);
129 extern void i_ldc_init_mapin(ldc_soft_state_t
*ldcssp
, uint64_t major
,
133 static ldc_ver_t ldc_versions
[] = { {1, 0} };
135 /* number of supported versions */
136 #define LDC_NUM_VERS (sizeof (ldc_versions) / sizeof (ldc_versions[0]))
138 /* Invalid value for the ldc_chan_t rx_ack_head field */
139 #define ACKPEEK_HEAD_INVALID ((uint64_t)-1)
142 /* Module State Pointer */
143 ldc_soft_state_t
*ldcssp
;
145 static struct modldrv md
= {
146 &mod_miscops
, /* This is a misc module */
147 "sun4v LDC module", /* Name of the module */
150 static struct modlinkage ml
= {
156 static uint64_t ldc_sup_minor
; /* Supported minor number */
157 static hsvc_info_t ldc_hsvc
= {
158 HSVC_REV_1
, NULL
, HSVC_GROUP_LDC
, 1, 2, "ldc"
162 * The no. of MTU size messages that can be stored in
163 * the LDC Tx queue. The number of Tx queue entries is
164 * then computed as (mtu * mtu_msgs)/sizeof(queue_entry)
166 uint64_t ldc_mtu_msgs
= LDC_MTU_MSGS
;
169 * The minimum queue length. This is the size of the smallest
170 * LDC queue. If the computed value is less than this default,
171 * the queue length is rounded up to 'ldc_queue_entries'.
173 uint64_t ldc_queue_entries
= LDC_QUEUE_ENTRIES
;
176 * The length of the reliable-mode data queue in terms of the LDC
177 * receive queue length. i.e., the number of times larger than the
178 * LDC receive queue that the data queue should be. The HV receive
179 * queue is required to be a power of 2 and this implementation
180 * assumes the data queue will also be a power of 2. By making the
181 * multiplier a power of 2, we ensure the data queue will be a
182 * power of 2. We use a multiplier because the receive queue is
183 * sized to be sane relative to the MTU and the same is needed for
186 uint64_t ldc_rxdq_multiplier
= LDC_RXDQ_MULTIPLIER
;
189 * LDC retry count and delay - when the HV returns EWOULDBLOCK
190 * the operation is retried 'ldc_max_retries' times with a
191 * wait of 'ldc_delay' usecs between each retry.
193 int ldc_max_retries
= LDC_MAX_RETRIES
;
194 clock_t ldc_delay
= LDC_DELAY
;
197 * Channels which have a devclass satisfying the following
198 * will be reset when entering the prom or kmdb.
200 * LDC_DEVCLASS_PROM_RESET(devclass) != 0
202 * By default, only block device service channels are reset.
204 #define LDC_DEVCLASS_BIT(dc) (0x1 << (dc))
205 #define LDC_DEVCLASS_PROM_RESET(dc) \
206 (LDC_DEVCLASS_BIT(dc) & ldc_debug_reset_mask)
207 static uint64_t ldc_debug_reset_mask
= LDC_DEVCLASS_BIT(LDC_DEV_BLK_SVC
) |
208 LDC_DEVCLASS_BIT(LDC_DEV_GENERIC
);
211 * delay between each retry of channel unregistration in
212 * ldc_close(), to wait for pending interrupts to complete.
214 clock_t ldc_close_delay
= LDC_CLOSE_DELAY
;
218 * Reserved mapin space for descriptor rings.
220 uint64_t ldc_dring_direct_map_rsvd
= LDC_DIRECT_MAP_SIZE_DEFAULT
;
223 * Maximum direct map space allowed per channel.
225 uint64_t ldc_direct_map_size_max
= (16 * 1024 * 1024); /* 16 MB */
230 * Print debug messages
232 * set ldcdbg to 0x7 for enabling all msgs
234 * 0x2 - All debug messages
235 * 0x1 - Minimal debug messages
237 * set ldcdbgchan to the channel number you want to debug
238 * setting it to -1 prints debug messages for all channels
239 * NOTE: ldcdbgchan has no effect on error messages
243 int64_t ldcdbgchan
= DBG_ALL_LDCS
;
244 uint64_t ldc_inject_err_flag
= 0;
247 ldcdebug(int64_t id
, const char *fmt
, ...)
254 * caller wants to print it anyway - (id == DBG_ALL_LDCS)
255 * debug channel is set to all LDCs - (ldcdbgchan == DBG_ALL_LDCS)
256 * debug channel = caller specified channel
258 if ((id
!= DBG_ALL_LDCS
) &&
259 (ldcdbgchan
!= DBG_ALL_LDCS
) &&
260 (ldcdbgchan
!= id
)) {
265 (void) vsprintf(buf
, fmt
, ap
);
268 cmn_err(CE_CONT
, "?%s", buf
);
271 #define LDC_ERR_RESET 0x1
272 #define LDC_ERR_PKTLOSS 0x2
273 #define LDC_ERR_DQFULL 0x4
274 #define LDC_ERR_DRNGCLEAR 0x8
277 ldc_inject_error(ldc_chan_t
*ldcp
, uint64_t error
)
279 if ((ldcdbgchan
!= DBG_ALL_LDCS
) && (ldcdbgchan
!= ldcp
->id
))
282 if ((ldc_inject_err_flag
& error
) == 0)
285 /* clear the injection state */
286 ldc_inject_err_flag
&= ~error
;
303 #define DUMP_PAYLOAD(id, addr) \
307 uint8_t *src = (uint8_t *)addr; \
308 for (i = 0; i < 64; i++, src++) \
309 (void) sprintf(&buf[i * 3], "|%02x", *src); \
310 (void) sprintf(&buf[i * 3], "|\n"); \
311 D2((id), "payload: %s", buf); \
314 #define DUMP_LDC_PKT(c, s, addr) \
316 ldc_msg_t *msg = (ldc_msg_t *)(addr); \
317 uint32_t mid = ((c)->mode != LDC_MODE_RAW) ? msg->seqid : 0; \
318 if (msg->type == LDC_DATA) { \
319 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env[%c%c,sz=%d])", \
320 (s), mid, msg->type, msg->stype, msg->ctrl, \
321 (msg->env & LDC_FRAG_START) ? 'B' : ' ', \
322 (msg->env & LDC_FRAG_STOP) ? 'E' : ' ', \
323 (msg->env & LDC_LEN_MASK)); \
325 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env=%x)", (s), \
326 mid, msg->type, msg->stype, msg->ctrl, msg->env); \
330 #define LDC_INJECT_RESET(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_RESET)
331 #define LDC_INJECT_PKTLOSS(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_PKTLOSS)
332 #define LDC_INJECT_DQFULL(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_DQFULL)
333 #define LDC_INJECT_DRNGCLEAR(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_DRNGCLEAR)
334 extern void i_ldc_mem_inject_dring_clear(ldc_chan_t
*ldcp
);
338 #define DBG_ALL_LDCS -1
344 #define DUMP_PAYLOAD(id, addr)
345 #define DUMP_LDC_PKT(c, s, addr)
347 #define LDC_INJECT_RESET(_ldcp) (B_FALSE)
348 #define LDC_INJECT_PKTLOSS(_ldcp) (B_FALSE)
349 #define LDC_INJECT_DQFULL(_ldcp) (B_FALSE)
350 #define LDC_INJECT_DRNGCLEAR(_ldcp) (B_FALSE)
355 * dtrace SDT probes to ease tracing of the rx data queue and HV queue
356 * lengths. Just pass the head, tail, and entries values so that the
357 * length can be calculated in a dtrace script when the probe is enabled.
359 #define TRACE_RXDQ_LENGTH(ldcp) \
360 DTRACE_PROBE4(rxdq__size, \
361 uint64_t, ldcp->id, \
362 uint64_t, ldcp->rx_dq_head, \
363 uint64_t, ldcp->rx_dq_tail, \
364 uint64_t, ldcp->rx_dq_entries)
366 #define TRACE_RXHVQ_LENGTH(ldcp, head, tail) \
367 DTRACE_PROBE4(rxhvq__size, \
368 uint64_t, ldcp->id, \
371 uint64_t, ldcp->rx_q_entries)
373 /* A dtrace SDT probe to ease tracing of data queue copy operations */
374 #define TRACE_RXDQ_COPY(ldcp, bytes) \
375 DTRACE_PROBE2(rxdq__copy, uint64_t, ldcp->id, uint64_t, bytes) \
377 /* The amount of contiguous space at the tail of the queue */
378 #define Q_CONTIG_SPACE(head, tail, size) \
379 ((head) <= (tail) ? ((size) - (tail)) : \
380 ((head) - (tail) - LDC_PACKET_SIZE))
382 #define ZERO_PKT(p) \
383 bzero((p), sizeof (ldc_msg_t));
385 #define IDX2COOKIE(idx, pg_szc, pg_shift) \
386 (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift)))
393 status
= hsvc_register(&ldc_hsvc
, &ldc_sup_minor
);
395 cmn_err(CE_NOTE
, "!%s: cannot negotiate hypervisor LDC services"
396 " group: 0x%lx major: %ld minor: %ld errno: %d",
397 ldc_hsvc
.hsvc_modname
, ldc_hsvc
.hsvc_group
,
398 ldc_hsvc
.hsvc_major
, ldc_hsvc
.hsvc_minor
, status
);
402 /* Initialize shared memory HV API version checking */
403 i_ldc_mem_set_hsvc_vers(ldc_hsvc
.hsvc_major
, ldc_sup_minor
);
405 /* allocate soft state structure */
406 ldcssp
= kmem_zalloc(sizeof (ldc_soft_state_t
), KM_SLEEP
);
408 i_ldc_init_mapin(ldcssp
, ldc_hsvc
.hsvc_major
, ldc_sup_minor
);
410 /* Link the module into the system */
411 status
= mod_install(&ml
);
413 kmem_free(ldcssp
, sizeof (ldc_soft_state_t
));
417 /* Initialize the LDC state structure */
418 mutex_init(&ldcssp
->lock
, NULL
, MUTEX_DRIVER
, NULL
);
420 mutex_enter(&ldcssp
->lock
);
422 /* Create a cache for memory handles */
423 ldcssp
->memhdl_cache
= kmem_cache_create("ldc_memhdl_cache",
424 sizeof (ldc_mhdl_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
425 if (ldcssp
->memhdl_cache
== NULL
) {
426 DWARN(DBG_ALL_LDCS
, "_init: ldc_memhdl cache create failed\n");
427 mutex_exit(&ldcssp
->lock
);
431 /* Create cache for memory segment structures */
432 ldcssp
->memseg_cache
= kmem_cache_create("ldc_memseg_cache",
433 sizeof (ldc_memseg_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
434 if (ldcssp
->memseg_cache
== NULL
) {
435 DWARN(DBG_ALL_LDCS
, "_init: ldc_memseg cache create failed\n");
436 mutex_exit(&ldcssp
->lock
);
441 ldcssp
->channel_count
= 0;
442 ldcssp
->channels_open
= 0;
443 ldcssp
->chan_list
= NULL
;
444 ldcssp
->dring_list
= NULL
;
446 /* Register debug_enter callback */
447 kldc_set_debug_cb(&i_ldc_debug_enter
);
449 mutex_exit(&ldcssp
->lock
);
455 _info(struct modinfo
*modinfop
)
457 /* Report status of the dynamically loadable driver module */
458 return (mod_info(&ml
, modinfop
));
465 ldc_chan_t
*tmp_ldcp
, *ldcp
;
466 ldc_dring_t
*tmp_dringp
, *dringp
;
467 ldc_mem_info_t minfo
;
469 /* Unlink the driver module from the system */
470 status
= mod_remove(&ml
);
472 DWARN(DBG_ALL_LDCS
, "_fini: mod_remove failed\n");
476 /* Unregister debug_enter callback */
477 kldc_set_debug_cb(NULL
);
479 /* Free descriptor rings */
480 dringp
= ldcssp
->dring_list
;
481 while (dringp
!= NULL
) {
482 tmp_dringp
= dringp
->next
;
484 rv
= ldc_mem_dring_info((ldc_dring_handle_t
)dringp
, &minfo
);
485 if (rv
== 0 && minfo
.status
!= LDC_UNBOUND
) {
486 if (minfo
.status
== LDC_BOUND
) {
487 (void) ldc_mem_dring_unbind(
488 (ldc_dring_handle_t
)dringp
);
490 if (minfo
.status
== LDC_MAPPED
) {
491 (void) ldc_mem_dring_unmap(
492 (ldc_dring_handle_t
)dringp
);
496 (void) ldc_mem_dring_destroy((ldc_dring_handle_t
)dringp
);
499 ldcssp
->dring_list
= NULL
;
501 /* close and finalize channels */
502 ldcp
= ldcssp
->chan_list
;
503 while (ldcp
!= NULL
) {
504 tmp_ldcp
= ldcp
->next
;
506 (void) ldc_close((ldc_handle_t
)ldcp
);
507 (void) ldc_fini((ldc_handle_t
)ldcp
);
511 ldcssp
->chan_list
= NULL
;
513 /* Destroy kmem caches */
514 kmem_cache_destroy(ldcssp
->memhdl_cache
);
515 kmem_cache_destroy(ldcssp
->memseg_cache
);
518 * We have successfully "removed" the driver.
519 * Destroying soft states
521 mutex_destroy(&ldcssp
->lock
);
522 kmem_free(ldcssp
, sizeof (ldc_soft_state_t
));
524 (void) hsvc_unregister(&ldc_hsvc
);
529 /* -------------------------------------------------------------------------- */
532 * LDC Link Layer Internal Functions
536 * Translate HV Errors to sun4v error codes
539 i_ldc_h2v_error(int h_error
)
554 return (EWOULDBLOCK
);
564 case H_ENOTSUPPORTED
:
580 * Reconfigure the transmit queue
583 i_ldc_txq_reconf(ldc_chan_t
*ldcp
)
587 ASSERT(MUTEX_HELD(&ldcp
->lock
));
588 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
590 rv
= hv_ldc_tx_qconf(ldcp
->id
, ldcp
->tx_q_ra
, ldcp
->tx_q_entries
);
593 "i_ldc_txq_reconf: (0x%lx) cannot set qconf", ldcp
->id
);
596 rv
= hv_ldc_tx_get_state(ldcp
->id
, &(ldcp
->tx_head
),
597 &(ldcp
->tx_tail
), &(ldcp
->link_state
));
600 "i_ldc_txq_reconf: (0x%lx) cannot get qptrs", ldcp
->id
);
603 D1(ldcp
->id
, "i_ldc_txq_reconf: (0x%llx) h=0x%llx,t=0x%llx,"
604 "s=0x%llx\n", ldcp
->id
, ldcp
->tx_head
, ldcp
->tx_tail
,
611 * Reconfigure the receive queue
614 i_ldc_rxq_reconf(ldc_chan_t
*ldcp
, boolean_t force_reset
)
617 uint64_t rx_head
, rx_tail
;
619 ASSERT(MUTEX_HELD(&ldcp
->lock
));
620 rv
= hv_ldc_rx_get_state(ldcp
->id
, &rx_head
, &rx_tail
,
621 &(ldcp
->link_state
));
624 "i_ldc_rxq_reconf: (0x%lx) cannot get state",
629 if (force_reset
|| (ldcp
->tstate
& ~TS_IN_RESET
) == TS_UP
) {
630 rv
= hv_ldc_rx_qconf(ldcp
->id
, ldcp
->rx_q_ra
,
634 "i_ldc_rxq_reconf: (0x%lx) cannot set qconf",
638 D1(ldcp
->id
, "i_ldc_rxq_reconf: (0x%llx) completed q reconf",
647 * Drain the contents of the receive queue
650 i_ldc_rxq_drain(ldc_chan_t
*ldcp
)
653 uint64_t rx_head
, rx_tail
;
656 ASSERT(MUTEX_HELD(&ldcp
->lock
));
657 rv
= hv_ldc_rx_get_state(ldcp
->id
, &rx_head
, &rx_tail
,
658 &(ldcp
->link_state
));
660 cmn_err(CE_WARN
, "i_ldc_rxq_drain: (0x%lx) cannot get state, "
661 "rv = 0x%x", ldcp
->id
, rv
);
665 /* If the queue is already empty just return success. */
666 if (rx_head
== rx_tail
)
670 * We are draining the queue in order to close the channel.
671 * Call hv_ldc_rx_set_qhead directly instead of i_ldc_set_rx_head
672 * because we do not need to reset the channel if the set
675 if ((rv
= hv_ldc_rx_set_qhead(ldcp
->id
, rx_tail
)) == 0)
678 while ((rv
== H_EWOULDBLOCK
) && (retries
++ < ldc_max_retries
)) {
679 drv_usecwait(ldc_delay
);
680 if ((rv
= hv_ldc_rx_set_qhead(ldcp
->id
, rx_tail
)) == 0)
684 cmn_err(CE_WARN
, "i_ldc_rxq_drain: (0x%lx) cannot set qhead 0x%lx, "
685 "rv = 0x%x", ldcp
->id
, rx_tail
, rv
);
690 * Reset LDC state structure and its contents
693 i_ldc_reset_state(ldc_chan_t
*ldcp
)
695 ASSERT(MUTEX_HELD(&ldcp
->lock
));
696 ldcp
->last_msg_snt
= LDC_INIT_SEQID
;
697 ldcp
->last_ack_rcd
= 0;
698 ldcp
->last_msg_rcd
= 0;
699 ldcp
->tx_ackd_head
= ldcp
->tx_head
;
700 ldcp
->stream_remains
= 0;
703 ldcp
->tstate
= TS_OPEN
;
704 ldcp
->status
= LDC_OPEN
;
705 ldcp
->rx_ack_head
= ACKPEEK_HEAD_INVALID
;
706 ldcp
->rx_dq_head
= 0;
707 ldcp
->rx_dq_tail
= 0;
709 if (ldcp
->link_state
== LDC_CHANNEL_UP
||
710 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
712 if (ldcp
->mode
== LDC_MODE_RAW
) {
713 ldcp
->status
= LDC_UP
;
714 ldcp
->tstate
= TS_UP
;
716 ldcp
->status
= LDC_READY
;
717 ldcp
->tstate
|= TS_LINK_READY
;
723 * Reset a LDC channel
726 i_ldc_reset(ldc_chan_t
*ldcp
, boolean_t force_reset
)
728 DWARN(ldcp
->id
, "i_ldc_reset: (0x%llx) channel reset\n", ldcp
->id
);
730 ASSERT(MUTEX_HELD(&ldcp
->lock
));
731 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
733 /* reconfig Tx and Rx queues */
734 (void) i_ldc_txq_reconf(ldcp
);
735 (void) i_ldc_rxq_reconf(ldcp
, force_reset
);
737 /* Clear Tx and Rx interrupts */
738 (void) i_ldc_clear_intr(ldcp
, CNEX_TX_INTR
);
739 (void) i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
741 /* Reset channel state */
742 i_ldc_reset_state(ldcp
);
744 /* Mark channel in reset */
745 ldcp
->tstate
|= TS_IN_RESET
;
749 * Walk the channel list and reset channels if they are of the right
750 * devclass and their Rx queues have been configured. No locks are
751 * taken because the function is only invoked by the kernel just before
752 * entering the prom or debugger when the system is single-threaded.
755 i_ldc_debug_enter(void)
759 ldcp
= ldcssp
->chan_list
;
760 while (ldcp
!= NULL
) {
761 if (((ldcp
->tstate
& TS_QCONF_RDY
) == TS_QCONF_RDY
) &&
762 (LDC_DEVCLASS_PROM_RESET(ldcp
->devclass
) != 0)) {
763 (void) hv_ldc_rx_qconf(ldcp
->id
, ldcp
->rx_q_ra
,
771 * Clear pending interrupts
774 i_ldc_clear_intr(ldc_chan_t
*ldcp
, cnex_intrtype_t itype
)
776 ldc_cnex_t
*cinfo
= &ldcssp
->cinfo
;
778 ASSERT(MUTEX_HELD(&ldcp
->lock
));
779 ASSERT(cinfo
->dip
!= NULL
);
783 /* check Tx interrupt */
784 if (ldcp
->tx_intr_state
)
785 ldcp
->tx_intr_state
= LDC_INTR_NONE
;
791 /* check Rx interrupt */
792 if (ldcp
->rx_intr_state
)
793 ldcp
->rx_intr_state
= LDC_INTR_NONE
;
799 (void) cinfo
->clr_intr(cinfo
->dip
, ldcp
->id
, itype
);
801 "i_ldc_clear_intr: (0x%llx) cleared 0x%x intr\n",
806 * Set the receive queue head
807 * Resets connection and returns an error if it fails.
810 i_ldc_set_rx_head(ldc_chan_t
*ldcp
, uint64_t head
)
815 ASSERT(MUTEX_HELD(&ldcp
->lock
));
816 for (retries
= 0; retries
< ldc_max_retries
; retries
++) {
818 if ((rv
= hv_ldc_rx_set_qhead(ldcp
->id
, head
)) == 0)
821 if (rv
!= H_EWOULDBLOCK
)
824 /* wait for ldc_delay usecs */
825 drv_usecwait(ldc_delay
);
828 cmn_err(CE_WARN
, "ldc_set_rx_qhead: (0x%lx) cannot set qhead 0x%lx, "
829 "rv = 0x%x", ldcp
->id
, head
, rv
);
830 mutex_enter(&ldcp
->tx_lock
);
831 i_ldc_reset(ldcp
, B_TRUE
);
832 mutex_exit(&ldcp
->tx_lock
);
838 * Returns the tx_head to be used for transfer
841 i_ldc_get_tx_head(ldc_chan_t
*ldcp
, uint64_t *head
)
845 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
847 /* get current Tx head */
848 *head
= ldcp
->tx_head
;
851 * Reliable mode will use the ACKd head instead of the regular tx_head.
852 * Also in Reliable mode, advance ackd_head for all non DATA/INFO pkts,
853 * up to the current location of tx_head. This needs to be done
854 * as the peer will only ACK DATA/INFO pkts.
856 if (ldcp
->mode
== LDC_MODE_RELIABLE
) {
857 while (ldcp
->tx_ackd_head
!= ldcp
->tx_head
) {
858 pkt
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ ldcp
->tx_ackd_head
);
859 if ((pkt
->type
& LDC_DATA
) && (pkt
->stype
& LDC_INFO
)) {
862 /* advance ACKd head */
864 (ldcp
->tx_ackd_head
+ LDC_PACKET_SIZE
) %
865 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
867 *head
= ldcp
->tx_ackd_head
;
872 * Returns the tx_tail to be used for transfer
873 * Re-reads the TX queue ptrs if and only if the
874 * the cached head and tail are equal (queue is full)
877 i_ldc_get_tx_tail(ldc_chan_t
*ldcp
, uint64_t *tail
)
880 uint64_t current_head
, new_tail
;
882 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
883 /* Read the head and tail ptrs from HV */
884 rv
= hv_ldc_tx_get_state(ldcp
->id
,
885 &ldcp
->tx_head
, &ldcp
->tx_tail
, &ldcp
->link_state
);
888 "i_ldc_get_tx_tail: (0x%lx) cannot read qptrs\n",
892 if (ldcp
->link_state
== LDC_CHANNEL_DOWN
) {
893 D1(ldcp
->id
, "i_ldc_get_tx_tail: (0x%llx) channel not ready\n",
898 i_ldc_get_tx_head(ldcp
, ¤t_head
);
900 /* increment the tail */
901 new_tail
= (ldcp
->tx_tail
+ LDC_PACKET_SIZE
) %
902 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
904 if (new_tail
== current_head
) {
906 "i_ldc_get_tx_tail: (0x%llx) TX queue is full\n",
908 return (EWOULDBLOCK
);
911 D2(ldcp
->id
, "i_ldc_get_tx_tail: (0x%llx) head=0x%llx, tail=0x%llx\n",
912 ldcp
->id
, ldcp
->tx_head
, ldcp
->tx_tail
);
914 *tail
= ldcp
->tx_tail
;
919 * Set the tail pointer. If HV returns EWOULDBLOCK, it will back off
920 * and retry ldc_max_retries times before returning an error.
921 * Returns 0, EWOULDBLOCK or EIO
924 i_ldc_set_tx_tail(ldc_chan_t
*ldcp
, uint64_t tail
)
926 int rv
, retval
= EWOULDBLOCK
;
929 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
930 for (retries
= 0; retries
< ldc_max_retries
; retries
++) {
932 if ((rv
= hv_ldc_tx_set_qtail(ldcp
->id
, tail
)) == 0) {
936 if (rv
!= H_EWOULDBLOCK
) {
937 DWARN(ldcp
->id
, "i_ldc_set_tx_tail: (0x%llx) set "
938 "qtail=0x%llx failed, rv=%d\n", ldcp
->id
, tail
, rv
);
943 /* wait for ldc_delay usecs */
944 drv_usecwait(ldc_delay
);
950 * Copy a data packet from the HV receive queue to the data queue.
951 * Caller must ensure that the data queue is not already full.
953 * The *head argument represents the current head pointer for the HV
954 * receive queue. After copying a packet from the HV receive queue,
955 * the *head pointer will be updated. This allows the caller to update
956 * the head pointer in HV using the returned *head value.
959 i_ldc_rxdq_copy(ldc_chan_t
*ldcp
, uint64_t *head
)
961 uint64_t q_size
, dq_size
;
963 ASSERT(MUTEX_HELD(&ldcp
->lock
));
965 q_size
= ldcp
->rx_q_entries
<< LDC_PACKET_SHIFT
;
966 dq_size
= ldcp
->rx_dq_entries
<< LDC_PACKET_SHIFT
;
968 ASSERT(Q_CONTIG_SPACE(ldcp
->rx_dq_head
, ldcp
->rx_dq_tail
,
969 dq_size
) >= LDC_PACKET_SIZE
);
971 bcopy((void *)(ldcp
->rx_q_va
+ *head
),
972 (void *)(ldcp
->rx_dq_va
+ ldcp
->rx_dq_tail
), LDC_PACKET_SIZE
);
973 TRACE_RXDQ_COPY(ldcp
, LDC_PACKET_SIZE
);
976 *head
= (*head
+ LDC_PACKET_SIZE
) % q_size
;
979 ldcp
->rx_dq_tail
= (ldcp
->rx_dq_tail
+ LDC_PACKET_SIZE
) % dq_size
;
983 * Update the Rx data queue head pointer
986 i_ldc_set_rxdq_head(ldc_chan_t
*ldcp
, uint64_t head
)
988 ldcp
->rx_dq_head
= head
;
993 * Get the Rx data queue head and tail pointers
996 i_ldc_dq_rx_get_state(ldc_chan_t
*ldcp
, uint64_t *head
, uint64_t *tail
,
997 uint64_t *link_state
)
999 _NOTE(ARGUNUSED(link_state
))
1000 *head
= ldcp
->rx_dq_head
;
1001 *tail
= ldcp
->rx_dq_tail
;
1006 * Wrapper for the Rx HV queue set head function. Giving the
1007 * data queue and HV queue set head functions the same type.
1010 i_ldc_hvq_rx_get_state(ldc_chan_t
*ldcp
, uint64_t *head
, uint64_t *tail
,
1011 uint64_t *link_state
)
1013 return (i_ldc_h2v_error(hv_ldc_rx_get_state(ldcp
->id
, head
, tail
,
1018 * LDC receive interrupt handler
1019 * triggered for channel with data pending to read
1020 * i.e. Rx queue content changes
1023 i_ldc_rx_hdlr(caddr_t arg1
, caddr_t arg2
)
1025 _NOTE(ARGUNUSED(arg2
))
1032 /* Get the channel for which interrupt was received */
1034 cmn_err(CE_WARN
, "i_ldc_rx_hdlr: invalid arg\n");
1035 return (DDI_INTR_UNCLAIMED
);
1038 ldcp
= (ldc_chan_t
*)arg1
;
1040 D1(ldcp
->id
, "i_ldc_rx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n",
1042 D1(ldcp
->id
, "i_ldc_rx_hdlr: (%llx) USR%lx/TS%lx/HS%lx, LSTATE=%lx\n",
1043 ldcp
->id
, ldcp
->status
, ldcp
->tstate
, ldcp
->hstate
,
1047 mutex_enter(&ldcp
->lock
);
1049 /* Mark the interrupt as being actively handled */
1050 ldcp
->rx_intr_state
= LDC_INTR_ACTIVE
;
1052 status
= i_ldc_rx_process_hvq(ldcp
, ¬ify
, &event
);
1054 if (ldcp
->mode
!= LDC_MODE_RELIABLE
) {
1056 * If there are no data packets on the queue, clear
1057 * the interrupt. Otherwise, the ldc_read will clear
1058 * interrupts after draining the queue. To indicate the
1059 * interrupt has not yet been cleared, it is marked
1062 if ((event
& LDC_EVT_READ
) == 0) {
1063 i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
1065 ldcp
->rx_intr_state
= LDC_INTR_PEND
;
1069 /* if callbacks are disabled, do not notify */
1070 if (notify
&& ldcp
->cb_enabled
) {
1071 ldcp
->cb_inprogress
= B_TRUE
;
1072 mutex_exit(&ldcp
->lock
);
1073 rv
= ldcp
->cb(event
, ldcp
->cb_arg
);
1076 "i_ldc_rx_hdlr: (0x%llx) callback failure",
1079 mutex_enter(&ldcp
->lock
);
1080 ldcp
->cb_inprogress
= B_FALSE
;
1083 if (ldcp
->mode
== LDC_MODE_RELIABLE
) {
1084 if (status
== ENOSPC
) {
1086 * Here, ENOSPC indicates the secondary data
1087 * queue is full and the Rx queue is non-empty.
1088 * Much like how reliable and raw modes are
1089 * handled above, since the Rx queue is non-
1090 * empty, we mark the interrupt as pending to
1091 * indicate it has not yet been cleared.
1093 ldcp
->rx_intr_state
= LDC_INTR_PEND
;
1096 * We have processed all CTRL packets and
1097 * copied all DATA packets to the secondary
1098 * queue. Clear the interrupt.
1100 i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
1104 mutex_exit(&ldcp
->lock
);
1106 D1(ldcp
->id
, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp
->id
);
1108 return (DDI_INTR_CLAIMED
);
1112 * Wrapper for the Rx HV queue processing function to be used when
1113 * checking the Rx HV queue for data packets. Unlike the interrupt
1114 * handler code flow, the Rx interrupt is not cleared here and
1115 * callbacks are not made.
1118 i_ldc_chkq(ldc_chan_t
*ldcp
)
1123 return (i_ldc_rx_process_hvq(ldcp
, ¬ify
, &event
));
1127 * Send a LDC message
1130 i_ldc_send_pkt(ldc_chan_t
*ldcp
, uint8_t pkttype
, uint8_t subtype
,
1136 uint32_t curr_seqid
;
1138 /* Obtain Tx lock */
1139 mutex_enter(&ldcp
->tx_lock
);
1141 curr_seqid
= ldcp
->last_msg_snt
;
1143 /* get the current tail for the message */
1144 rv
= i_ldc_get_tx_tail(ldcp
, &tx_tail
);
1147 "i_ldc_send_pkt: (0x%llx) error sending pkt, "
1148 "type=0x%x,subtype=0x%x,ctrl=0x%x\n",
1149 ldcp
->id
, pkttype
, subtype
, ctrlmsg
);
1150 mutex_exit(&ldcp
->tx_lock
);
1154 pkt
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
1157 /* Initialize the packet */
1158 pkt
->type
= pkttype
;
1159 pkt
->stype
= subtype
;
1160 pkt
->ctrl
= ctrlmsg
;
1162 /* Store ackid/seqid iff it is RELIABLE mode & not a RTS/RTR message */
1163 if (((ctrlmsg
& LDC_CTRL_MASK
) != LDC_RTS
) &&
1164 ((ctrlmsg
& LDC_CTRL_MASK
) != LDC_RTR
)) {
1166 if (ldcp
->mode
!= LDC_MODE_RAW
) {
1167 pkt
->seqid
= curr_seqid
;
1168 pkt
->ackid
= ldcp
->last_msg_rcd
;
1171 DUMP_LDC_PKT(ldcp
, "i_ldc_send_pkt", (uint64_t)pkt
);
1173 /* initiate the send by calling into HV and set the new tail */
1174 tx_tail
= (tx_tail
+ LDC_PACKET_SIZE
) %
1175 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
1177 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
1180 "i_ldc_send_pkt:(0x%llx) error sending pkt, "
1181 "type=0x%x,stype=0x%x,ctrl=0x%x\n",
1182 ldcp
->id
, pkttype
, subtype
, ctrlmsg
);
1183 mutex_exit(&ldcp
->tx_lock
);
1187 ldcp
->last_msg_snt
= curr_seqid
;
1188 ldcp
->tx_tail
= tx_tail
;
1190 mutex_exit(&ldcp
->tx_lock
);
1195 * Checks if packet was received in right order
1196 * in the case of a reliable link.
1197 * Returns 0 if in order, else EIO
1200 i_ldc_check_seqid(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
)
1202 /* No seqid checking for RAW mode */
1203 if (ldcp
->mode
== LDC_MODE_RAW
)
1206 /* No seqid checking for version, RTS, RTR message */
1207 if (msg
->ctrl
== LDC_VER
||
1208 msg
->ctrl
== LDC_RTS
||
1209 msg
->ctrl
== LDC_RTR
)
1212 /* Initial seqid to use is sent in RTS/RTR and saved in last_msg_rcd */
1213 if (msg
->seqid
!= (ldcp
->last_msg_rcd
+ 1)) {
1215 "i_ldc_check_seqid: (0x%llx) out-of-order pkt, got 0x%x, "
1216 "expecting 0x%x\n", ldcp
->id
, msg
->seqid
,
1217 (ldcp
->last_msg_rcd
+ 1));
1222 if (LDC_INJECT_PKTLOSS(ldcp
)) {
1224 "i_ldc_check_seqid: (0x%llx) inject pkt loss\n", ldcp
->id
);
1234 * Process an incoming version ctrl message
1237 i_ldc_process_VER(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
)
1239 int rv
= 0, idx
= ldcp
->next_vidx
;
1242 ldc_ver_t
*rcvd_ver
;
1244 /* get the received version */
1245 rcvd_ver
= (ldc_ver_t
*)((uint64_t)msg
+ LDC_PAYLOAD_VER_OFF
);
1247 D2(ldcp
->id
, "i_ldc_process_VER: (0x%llx) received VER v%u.%u\n",
1248 ldcp
->id
, rcvd_ver
->major
, rcvd_ver
->minor
);
1250 /* Obtain Tx lock */
1251 mutex_enter(&ldcp
->tx_lock
);
1253 switch (msg
->stype
) {
1256 if ((ldcp
->tstate
& ~TS_IN_RESET
) == TS_VREADY
) {
1257 (void) i_ldc_txq_reconf(ldcp
);
1258 i_ldc_reset_state(ldcp
);
1259 mutex_exit(&ldcp
->tx_lock
);
1263 /* get the current tail and pkt for the response */
1264 rv
= i_ldc_get_tx_tail(ldcp
, &tx_tail
);
1267 "i_ldc_process_VER: (0x%llx) err sending "
1268 "version ACK/NACK\n", ldcp
->id
);
1269 i_ldc_reset(ldcp
, B_TRUE
);
1270 mutex_exit(&ldcp
->tx_lock
);
1271 return (ECONNRESET
);
1274 pkt
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
1277 /* initialize the packet */
1278 pkt
->type
= LDC_CTRL
;
1279 pkt
->ctrl
= LDC_VER
;
1283 D1(ldcp
->id
, "i_ldc_process_VER: got %u.%u chk %u.%u\n",
1284 rcvd_ver
->major
, rcvd_ver
->minor
,
1285 ldc_versions
[idx
].major
, ldc_versions
[idx
].minor
);
1287 if (rcvd_ver
->major
== ldc_versions
[idx
].major
) {
1288 /* major version match - ACK version */
1289 pkt
->stype
= LDC_ACK
;
1292 * lower minor version to the one this endpt
1293 * supports, if necessary
1295 if (rcvd_ver
->minor
> ldc_versions
[idx
].minor
)
1297 ldc_versions
[idx
].minor
;
1298 bcopy(rcvd_ver
, pkt
->udata
, sizeof (*rcvd_ver
));
1303 if (rcvd_ver
->major
> ldc_versions
[idx
].major
) {
1305 D1(ldcp
->id
, "i_ldc_process_VER: using next"
1306 " lower idx=%d, v%u.%u\n", idx
,
1307 ldc_versions
[idx
].major
,
1308 ldc_versions
[idx
].minor
);
1310 /* nack with next lower version */
1311 pkt
->stype
= LDC_NACK
;
1312 bcopy(&ldc_versions
[idx
], pkt
->udata
,
1313 sizeof (ldc_versions
[idx
]));
1314 ldcp
->next_vidx
= idx
;
1318 /* next major version */
1321 D1(ldcp
->id
, "i_ldc_process_VER: inc idx %x\n", idx
);
1323 if (idx
== LDC_NUM_VERS
) {
1324 /* no version match - send NACK */
1325 pkt
->stype
= LDC_NACK
;
1326 bzero(pkt
->udata
, sizeof (ldc_ver_t
));
1327 ldcp
->next_vidx
= 0;
1332 /* initiate the send by calling into HV and set the new tail */
1333 tx_tail
= (tx_tail
+ LDC_PACKET_SIZE
) %
1334 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
1336 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
1338 ldcp
->tx_tail
= tx_tail
;
1339 if (pkt
->stype
== LDC_ACK
) {
1340 D2(ldcp
->id
, "i_ldc_process_VER: (0x%llx) sent"
1341 " version ACK\n", ldcp
->id
);
1342 /* Save the ACK'd version */
1343 ldcp
->version
.major
= rcvd_ver
->major
;
1344 ldcp
->version
.minor
= rcvd_ver
->minor
;
1345 ldcp
->hstate
|= TS_RCVD_VER
;
1346 ldcp
->tstate
|= TS_VER_DONE
;
1348 "(0x%llx) Sent ACK, "
1349 "Agreed on version v%u.%u\n",
1350 ldcp
->id
, rcvd_ver
->major
, rcvd_ver
->minor
);
1354 "i_ldc_process_VER: (0x%llx) error sending "
1355 "ACK/NACK\n", ldcp
->id
);
1356 i_ldc_reset(ldcp
, B_TRUE
);
1357 mutex_exit(&ldcp
->tx_lock
);
1358 return (ECONNRESET
);
1364 if ((ldcp
->tstate
& ~TS_IN_RESET
) == TS_VREADY
) {
1365 if (ldcp
->version
.major
!= rcvd_ver
->major
||
1366 ldcp
->version
.minor
!= rcvd_ver
->minor
) {
1368 /* mismatched version - reset connection */
1370 "i_ldc_process_VER: (0x%llx) recvd"
1371 " ACK ver != sent ACK ver\n", ldcp
->id
);
1372 i_ldc_reset(ldcp
, B_TRUE
);
1373 mutex_exit(&ldcp
->tx_lock
);
1374 return (ECONNRESET
);
1377 /* SUCCESS - we have agreed on a version */
1378 ldcp
->version
.major
= rcvd_ver
->major
;
1379 ldcp
->version
.minor
= rcvd_ver
->minor
;
1380 ldcp
->tstate
|= TS_VER_DONE
;
1383 D1(ldcp
->id
, "(0x%llx) Got ACK, Agreed on version v%u.%u\n",
1384 ldcp
->id
, rcvd_ver
->major
, rcvd_ver
->minor
);
1386 /* initiate RTS-RTR-RDX handshake */
1387 rv
= i_ldc_get_tx_tail(ldcp
, &tx_tail
);
1390 "i_ldc_process_VER: (0x%llx) cannot send RTS\n",
1392 i_ldc_reset(ldcp
, B_TRUE
);
1393 mutex_exit(&ldcp
->tx_lock
);
1394 return (ECONNRESET
);
1397 pkt
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
1400 pkt
->type
= LDC_CTRL
;
1401 pkt
->stype
= LDC_INFO
;
1402 pkt
->ctrl
= LDC_RTS
;
1403 pkt
->env
= ldcp
->mode
;
1404 if (ldcp
->mode
!= LDC_MODE_RAW
)
1405 pkt
->seqid
= LDC_INIT_SEQID
;
1407 ldcp
->last_msg_rcd
= LDC_INIT_SEQID
;
1409 DUMP_LDC_PKT(ldcp
, "i_ldc_process_VER snd rts", (uint64_t)pkt
);
1411 /* initiate the send by calling into HV and set the new tail */
1412 tx_tail
= (tx_tail
+ LDC_PACKET_SIZE
) %
1413 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
1415 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
1418 "i_ldc_process_VER: (0x%llx) no listener\n",
1420 i_ldc_reset(ldcp
, B_TRUE
);
1421 mutex_exit(&ldcp
->tx_lock
);
1422 return (ECONNRESET
);
1425 ldcp
->tx_tail
= tx_tail
;
1426 ldcp
->hstate
|= TS_SENT_RTS
;
1431 /* check if version in NACK is zero */
1432 if (rcvd_ver
->major
== 0 && rcvd_ver
->minor
== 0) {
1433 /* version handshake failure */
1435 "i_ldc_process_VER: (0x%llx) no version match\n",
1437 i_ldc_reset(ldcp
, B_TRUE
);
1438 mutex_exit(&ldcp
->tx_lock
);
1439 return (ECONNRESET
);
1442 /* get the current tail and pkt for the response */
1443 rv
= i_ldc_get_tx_tail(ldcp
, &tx_tail
);
1446 "i_ldc_process_VER: (0x%lx) err sending "
1447 "version ACK/NACK\n", ldcp
->id
);
1448 i_ldc_reset(ldcp
, B_TRUE
);
1449 mutex_exit(&ldcp
->tx_lock
);
1450 return (ECONNRESET
);
1453 pkt
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
1456 /* initialize the packet */
1457 pkt
->type
= LDC_CTRL
;
1458 pkt
->ctrl
= LDC_VER
;
1459 pkt
->stype
= LDC_INFO
;
1461 /* check ver in NACK msg has a match */
1463 if (rcvd_ver
->major
== ldc_versions
[idx
].major
) {
1465 * major version match - resubmit request
1466 * if lower minor version to the one this endpt
1467 * supports, if necessary
1469 if (rcvd_ver
->minor
> ldc_versions
[idx
].minor
)
1471 ldc_versions
[idx
].minor
;
1472 bcopy(rcvd_ver
, pkt
->udata
, sizeof (*rcvd_ver
));
1476 if (rcvd_ver
->major
> ldc_versions
[idx
].major
) {
1478 D1(ldcp
->id
, "i_ldc_process_VER: using next"
1479 " lower idx=%d, v%u.%u\n", idx
,
1480 ldc_versions
[idx
].major
,
1481 ldc_versions
[idx
].minor
);
1483 /* send next lower version */
1484 bcopy(&ldc_versions
[idx
], pkt
->udata
,
1485 sizeof (ldc_versions
[idx
]));
1486 ldcp
->next_vidx
= idx
;
1493 D1(ldcp
->id
, "i_ldc_process_VER: inc idx %x\n", idx
);
1495 if (idx
== LDC_NUM_VERS
) {
1496 /* no version match - terminate */
1497 ldcp
->next_vidx
= 0;
1498 mutex_exit(&ldcp
->tx_lock
);
1499 return (ECONNRESET
);
1503 /* initiate the send by calling into HV and set the new tail */
1504 tx_tail
= (tx_tail
+ LDC_PACKET_SIZE
) %
1505 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
1507 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
1509 D2(ldcp
->id
, "i_ldc_process_VER: (0x%llx) sent version"
1510 "INFO v%u.%u\n", ldcp
->id
, ldc_versions
[idx
].major
,
1511 ldc_versions
[idx
].minor
);
1512 ldcp
->tx_tail
= tx_tail
;
1515 "i_ldc_process_VER: (0x%lx) error sending version"
1516 "INFO\n", ldcp
->id
);
1517 i_ldc_reset(ldcp
, B_TRUE
);
1518 mutex_exit(&ldcp
->tx_lock
);
1519 return (ECONNRESET
);
1525 mutex_exit(&ldcp
->tx_lock
);
1531 * Process an incoming RTS ctrl message
1534 i_ldc_process_RTS(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
)
1539 boolean_t sent_NACK
= B_FALSE
;
1541 D2(ldcp
->id
, "i_ldc_process_RTS: (0x%llx) received RTS\n", ldcp
->id
);
1543 switch (msg
->stype
) {
1546 "i_ldc_process_RTS: (0x%llx) RTS NACK received\n",
1549 /* Reset the channel -- as we cannot continue */
1550 mutex_enter(&ldcp
->tx_lock
);
1551 i_ldc_reset(ldcp
, B_TRUE
);
1552 mutex_exit(&ldcp
->tx_lock
);
1559 if (ldcp
->mode
!= (ldc_mode_t
)msg
->env
) {
1561 "i_ldc_process_RTS: (0x%lx) mode mismatch\n",
1564 * send NACK in response to MODE message
1565 * get the current tail for the response
1567 rv
= i_ldc_send_pkt(ldcp
, LDC_CTRL
, LDC_NACK
, LDC_RTS
);
1569 /* if cannot send NACK - reset channel */
1570 mutex_enter(&ldcp
->tx_lock
);
1571 i_ldc_reset(ldcp
, B_TRUE
);
1572 mutex_exit(&ldcp
->tx_lock
);
1580 DWARN(ldcp
->id
, "i_ldc_process_RTS: (0x%llx) unexp ACK\n",
1582 mutex_enter(&ldcp
->tx_lock
);
1583 i_ldc_reset(ldcp
, B_TRUE
);
1584 mutex_exit(&ldcp
->tx_lock
);
1590 * If either the connection was reset (when rv != 0) or
1591 * a NACK was sent, we return. In the case of a NACK
1592 * we dont want to consume the packet that came in but
1593 * not record that we received the RTS
1595 if (rv
|| sent_NACK
)
1598 /* record RTS received */
1599 ldcp
->hstate
|= TS_RCVD_RTS
;
1601 /* store initial SEQID info */
1602 ldcp
->last_msg_snt
= msg
->seqid
;
1604 /* Obtain Tx lock */
1605 mutex_enter(&ldcp
->tx_lock
);
1607 /* get the current tail for the response */
1608 rv
= i_ldc_get_tx_tail(ldcp
, &tx_tail
);
1611 "i_ldc_process_RTS: (0x%lx) err sending RTR\n",
1613 i_ldc_reset(ldcp
, B_TRUE
);
1614 mutex_exit(&ldcp
->tx_lock
);
1615 return (ECONNRESET
);
1618 pkt
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
1621 /* initialize the packet */
1622 pkt
->type
= LDC_CTRL
;
1623 pkt
->stype
= LDC_INFO
;
1624 pkt
->ctrl
= LDC_RTR
;
1625 pkt
->env
= ldcp
->mode
;
1626 if (ldcp
->mode
!= LDC_MODE_RAW
)
1627 pkt
->seqid
= LDC_INIT_SEQID
;
1629 ldcp
->last_msg_rcd
= msg
->seqid
;
1631 /* initiate the send by calling into HV and set the new tail */
1632 tx_tail
= (tx_tail
+ LDC_PACKET_SIZE
) %
1633 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
1635 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
1638 "i_ldc_process_RTS: (0x%llx) sent RTR\n", ldcp
->id
);
1639 DUMP_LDC_PKT(ldcp
, "i_ldc_process_RTS sent rtr", (uint64_t)pkt
);
1641 ldcp
->tx_tail
= tx_tail
;
1642 ldcp
->hstate
|= TS_SENT_RTR
;
1646 "i_ldc_process_RTS: (0x%lx) error sending RTR\n",
1648 i_ldc_reset(ldcp
, B_TRUE
);
1649 mutex_exit(&ldcp
->tx_lock
);
1650 return (ECONNRESET
);
1653 mutex_exit(&ldcp
->tx_lock
);
1658 * Process an incoming RTR ctrl message
1661 i_ldc_process_RTR(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
)
1664 boolean_t sent_NACK
= B_FALSE
;
1666 D2(ldcp
->id
, "i_ldc_process_RTR: (0x%llx) received RTR\n", ldcp
->id
);
1668 switch (msg
->stype
) {
1670 /* RTR NACK received */
1672 "i_ldc_process_RTR: (0x%llx) RTR NACK received\n",
1675 /* Reset the channel -- as we cannot continue */
1676 mutex_enter(&ldcp
->tx_lock
);
1677 i_ldc_reset(ldcp
, B_TRUE
);
1678 mutex_exit(&ldcp
->tx_lock
);
1686 if (ldcp
->mode
!= (ldc_mode_t
)msg
->env
) {
1688 "i_ldc_process_RTR: (0x%llx) mode mismatch, "
1689 "expecting 0x%x, got 0x%x\n",
1690 ldcp
->id
, ldcp
->mode
, (ldc_mode_t
)msg
->env
);
1692 * send NACK in response to MODE message
1693 * get the current tail for the response
1695 rv
= i_ldc_send_pkt(ldcp
, LDC_CTRL
, LDC_NACK
, LDC_RTR
);
1697 /* if cannot send NACK - reset channel */
1698 mutex_enter(&ldcp
->tx_lock
);
1699 i_ldc_reset(ldcp
, B_TRUE
);
1700 mutex_exit(&ldcp
->tx_lock
);
1709 DWARN(ldcp
->id
, "i_ldc_process_RTR: (0x%llx) unexp ACK\n",
1712 /* Reset the channel -- as we cannot continue */
1713 mutex_enter(&ldcp
->tx_lock
);
1714 i_ldc_reset(ldcp
, B_TRUE
);
1715 mutex_exit(&ldcp
->tx_lock
);
1721 * If either the connection was reset (when rv != 0) or
1722 * a NACK was sent, we return. In the case of a NACK
1723 * we dont want to consume the packet that came in but
1724 * not record that we received the RTR
1726 if (rv
|| sent_NACK
)
1729 ldcp
->last_msg_snt
= msg
->seqid
;
1730 ldcp
->hstate
|= TS_RCVD_RTR
;
1732 rv
= i_ldc_send_pkt(ldcp
, LDC_CTRL
, LDC_INFO
, LDC_RDX
);
1735 "i_ldc_process_RTR: (0x%lx) cannot send RDX\n",
1737 mutex_enter(&ldcp
->tx_lock
);
1738 i_ldc_reset(ldcp
, B_TRUE
);
1739 mutex_exit(&ldcp
->tx_lock
);
1740 return (ECONNRESET
);
1743 "i_ldc_process_RTR: (0x%llx) sent RDX\n", ldcp
->id
);
1745 ldcp
->hstate
|= TS_SENT_RDX
;
1746 ldcp
->tstate
|= TS_HSHAKE_DONE
;
1747 if ((ldcp
->tstate
& TS_IN_RESET
) == 0)
1748 ldcp
->status
= LDC_UP
;
1750 D1(ldcp
->id
, "(0x%llx) Handshake Complete\n", ldcp
->id
);
1757 * Process an incoming RDX ctrl message
1760 i_ldc_process_RDX(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
)
1764 D2(ldcp
->id
, "i_ldc_process_RDX: (0x%llx) received RDX\n", ldcp
->id
);
1766 switch (msg
->stype
) {
1768 /* RDX NACK received */
1770 "i_ldc_process_RDX: (0x%llx) RDX NACK received\n",
1773 /* Reset the channel -- as we cannot continue */
1774 mutex_enter(&ldcp
->tx_lock
);
1775 i_ldc_reset(ldcp
, B_TRUE
);
1776 mutex_exit(&ldcp
->tx_lock
);
1784 * if channel is UP and a RDX received after data transmission
1785 * has commenced it is an error
1787 if ((ldcp
->tstate
== TS_UP
) && (ldcp
->hstate
& TS_RCVD_RDX
)) {
1789 "i_ldc_process_RDX: (0x%llx) unexpected RDX"
1790 " - LDC reset\n", ldcp
->id
);
1791 mutex_enter(&ldcp
->tx_lock
);
1792 i_ldc_reset(ldcp
, B_TRUE
);
1793 mutex_exit(&ldcp
->tx_lock
);
1794 return (ECONNRESET
);
1797 ldcp
->hstate
|= TS_RCVD_RDX
;
1798 ldcp
->tstate
|= TS_HSHAKE_DONE
;
1799 if ((ldcp
->tstate
& TS_IN_RESET
) == 0)
1800 ldcp
->status
= LDC_UP
;
1802 D1(DBG_ALL_LDCS
, "(0x%llx) Handshake Complete\n", ldcp
->id
);
1806 DWARN(ldcp
->id
, "i_ldc_process_RDX: (0x%llx) unexp ACK\n",
1809 /* Reset the channel -- as we cannot continue */
1810 mutex_enter(&ldcp
->tx_lock
);
1811 i_ldc_reset(ldcp
, B_TRUE
);
1812 mutex_exit(&ldcp
->tx_lock
);
1821 * Process an incoming ACK for a data packet
1824 i_ldc_process_data_ACK(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
)
1830 /* Obtain Tx lock */
1831 mutex_enter(&ldcp
->tx_lock
);
1834 * Read the current Tx head and tail
1836 rv
= hv_ldc_tx_get_state(ldcp
->id
,
1837 &ldcp
->tx_head
, &ldcp
->tx_tail
, &ldcp
->link_state
);
1840 "i_ldc_process_data_ACK: (0x%lx) cannot read qptrs\n",
1843 /* Reset the channel -- as we cannot continue */
1844 i_ldc_reset(ldcp
, B_TRUE
);
1845 mutex_exit(&ldcp
->tx_lock
);
1846 return (ECONNRESET
);
1850 * loop from where the previous ACK location was to the
1851 * current head location. This is how far the HV has
1852 * actually send pkts. Pkts between head and tail are
1853 * yet to be sent by HV.
1855 tx_head
= ldcp
->tx_ackd_head
;
1857 pkt
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_head
);
1858 tx_head
= (tx_head
+ LDC_PACKET_SIZE
) %
1859 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
1861 if (pkt
->seqid
== msg
->ackid
) {
1863 "i_ldc_process_data_ACK: (0x%llx) found packet\n",
1865 ldcp
->last_ack_rcd
= msg
->ackid
;
1866 ldcp
->tx_ackd_head
= tx_head
;
1869 if (tx_head
== ldcp
->tx_head
) {
1870 /* could not find packet */
1872 "i_ldc_process_data_ACK: (0x%llx) invalid ACKid\n",
1875 /* Reset the channel -- as we cannot continue */
1876 i_ldc_reset(ldcp
, B_TRUE
);
1877 mutex_exit(&ldcp
->tx_lock
);
1878 return (ECONNRESET
);
1882 mutex_exit(&ldcp
->tx_lock
);
1887 * Process incoming control message
1888 * Return 0 - session can continue
1889 * EAGAIN - reprocess packet - state was changed
1890 * ECONNRESET - channel was reset
1893 i_ldc_ctrlmsg(ldc_chan_t
*ldcp
, ldc_msg_t
*msg
)
1897 D1(ldcp
->id
, "i_ldc_ctrlmsg: (%llx) tstate = %lx, hstate = %lx\n",
1898 ldcp
->id
, ldcp
->tstate
, ldcp
->hstate
);
1900 switch (ldcp
->tstate
& ~TS_IN_RESET
) {
1905 switch (msg
->ctrl
& LDC_CTRL_MASK
) {
1907 /* process version message */
1908 rv
= i_ldc_process_VER(ldcp
, msg
);
1912 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x "
1913 "tstate=0x%x\n", ldcp
->id
,
1914 (msg
->ctrl
& LDC_CTRL_MASK
), ldcp
->tstate
);
1922 switch (msg
->ctrl
& LDC_CTRL_MASK
) {
1924 /* process version message */
1925 rv
= i_ldc_process_VER(ldcp
, msg
);
1928 /* process RTS message */
1929 rv
= i_ldc_process_RTS(ldcp
, msg
);
1932 /* process RTR message */
1933 rv
= i_ldc_process_RTR(ldcp
, msg
);
1936 /* process RDX message */
1937 rv
= i_ldc_process_RDX(ldcp
, msg
);
1941 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x "
1942 "tstate=0x%x\n", ldcp
->id
,
1943 (msg
->ctrl
& LDC_CTRL_MASK
), ldcp
->tstate
);
1951 switch (msg
->ctrl
& LDC_CTRL_MASK
) {
1954 "i_ldc_ctrlmsg: (0x%llx) unexpected VER "
1955 "- LDC reset\n", ldcp
->id
);
1956 /* peer is redoing version negotiation */
1957 mutex_enter(&ldcp
->tx_lock
);
1958 (void) i_ldc_txq_reconf(ldcp
);
1959 i_ldc_reset_state(ldcp
);
1960 mutex_exit(&ldcp
->tx_lock
);
1965 /* process RDX message */
1966 rv
= i_ldc_process_RDX(ldcp
, msg
);
1971 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x "
1972 "tstate=0x%x\n", ldcp
->id
,
1973 (msg
->ctrl
& LDC_CTRL_MASK
), ldcp
->tstate
);
1982 * Register channel with the channel nexus
1985 i_ldc_register_channel(ldc_chan_t
*ldcp
)
1988 ldc_cnex_t
*cinfo
= &ldcssp
->cinfo
;
1990 if (cinfo
->dip
== NULL
) {
1992 "i_ldc_register_channel: cnex has not registered\n");
1996 rv
= cinfo
->reg_chan(cinfo
->dip
, ldcp
->id
, ldcp
->devclass
);
1999 "i_ldc_register_channel: cannot register channel\n");
2003 rv
= cinfo
->add_intr(cinfo
->dip
, ldcp
->id
, CNEX_TX_INTR
,
2004 i_ldc_tx_hdlr
, ldcp
, NULL
);
2007 "i_ldc_register_channel: cannot add Tx interrupt\n");
2008 (void) cinfo
->unreg_chan(cinfo
->dip
, ldcp
->id
);
2012 rv
= cinfo
->add_intr(cinfo
->dip
, ldcp
->id
, CNEX_RX_INTR
,
2013 i_ldc_rx_hdlr
, ldcp
, NULL
);
2016 "i_ldc_register_channel: cannot add Rx interrupt\n");
2017 (void) cinfo
->rem_intr(cinfo
->dip
, ldcp
->id
, CNEX_TX_INTR
);
2018 (void) cinfo
->unreg_chan(cinfo
->dip
, ldcp
->id
);
2022 ldcp
->tstate
|= TS_CNEX_RDY
;
2028 * Unregister a channel with the channel nexus
2031 i_ldc_unregister_channel(ldc_chan_t
*ldcp
)
2034 ldc_cnex_t
*cinfo
= &ldcssp
->cinfo
;
2036 if (cinfo
->dip
== NULL
) {
2038 "i_ldc_unregister_channel: cnex has not registered\n");
2042 if (ldcp
->tstate
& TS_CNEX_RDY
) {
2044 /* Remove the Rx interrupt */
2045 rv
= cinfo
->rem_intr(cinfo
->dip
, ldcp
->id
, CNEX_RX_INTR
);
2049 "i_ldc_unregister_channel: err removing "
2055 * If interrupts are pending and handler has
2056 * finished running, clear interrupt and try
2059 if (ldcp
->rx_intr_state
!= LDC_INTR_PEND
)
2062 (void) i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
2063 rv
= cinfo
->rem_intr(cinfo
->dip
, ldcp
->id
,
2066 DWARN(ldcp
->id
, "i_ldc_unregister_channel: "
2067 "err removing Rx interrupt\n");
2072 /* Remove the Tx interrupt */
2073 rv
= cinfo
->rem_intr(cinfo
->dip
, ldcp
->id
, CNEX_TX_INTR
);
2076 "i_ldc_unregister_channel: err removing Tx intr\n");
2080 /* Unregister the channel */
2081 rv
= cinfo
->unreg_chan(ldcssp
->cinfo
.dip
, ldcp
->id
);
2084 "i_ldc_unregister_channel: cannot unreg channel\n");
2088 ldcp
->tstate
&= ~TS_CNEX_RDY
;
2096 * LDC transmit interrupt handler
2097 * triggered for chanel up/down/reset events
2098 * and Tx queue content changes
2101 i_ldc_tx_hdlr(caddr_t arg1
, caddr_t arg2
)
2103 _NOTE(ARGUNUSED(arg2
))
2107 boolean_t notify_client
= B_FALSE
;
2108 uint64_t notify_event
= 0, link_state
;
2110 /* Get the channel for which interrupt was received */
2111 ASSERT(arg1
!= NULL
);
2112 ldcp
= (ldc_chan_t
*)arg1
;
2114 D1(ldcp
->id
, "i_ldc_tx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n",
2118 mutex_enter(&ldcp
->lock
);
2120 /* Obtain Tx lock */
2121 mutex_enter(&ldcp
->tx_lock
);
2123 /* mark interrupt as pending */
2124 ldcp
->tx_intr_state
= LDC_INTR_ACTIVE
;
2126 /* save current link state */
2127 link_state
= ldcp
->link_state
;
2129 rv
= hv_ldc_tx_get_state(ldcp
->id
, &ldcp
->tx_head
, &ldcp
->tx_tail
,
2133 "i_ldc_tx_hdlr: (0x%lx) cannot read queue ptrs rv=0x%d\n",
2135 i_ldc_clear_intr(ldcp
, CNEX_TX_INTR
);
2136 mutex_exit(&ldcp
->tx_lock
);
2137 mutex_exit(&ldcp
->lock
);
2138 return (DDI_INTR_CLAIMED
);
2142 * reset the channel state if the channel went down
2143 * (other side unconfigured queue) or channel was reset
2144 * (other side reconfigured its queue)
2146 if (link_state
!= ldcp
->link_state
&&
2147 ldcp
->link_state
== LDC_CHANNEL_DOWN
) {
2148 D1(ldcp
->id
, "i_ldc_tx_hdlr: channel link down\n", ldcp
->id
);
2149 i_ldc_reset(ldcp
, B_FALSE
);
2150 notify_client
= B_TRUE
;
2151 notify_event
= LDC_EVT_DOWN
;
2154 if (link_state
!= ldcp
->link_state
&&
2155 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
2156 D1(ldcp
->id
, "i_ldc_tx_hdlr: channel link reset\n", ldcp
->id
);
2157 i_ldc_reset(ldcp
, B_FALSE
);
2158 notify_client
= B_TRUE
;
2159 notify_event
= LDC_EVT_RESET
;
2162 if (link_state
!= ldcp
->link_state
&&
2163 (ldcp
->tstate
& ~TS_IN_RESET
) == TS_OPEN
&&
2164 ldcp
->link_state
== LDC_CHANNEL_UP
) {
2165 D1(ldcp
->id
, "i_ldc_tx_hdlr: channel link up\n", ldcp
->id
);
2166 notify_client
= B_TRUE
;
2167 notify_event
= LDC_EVT_RESET
;
2168 ldcp
->tstate
|= TS_LINK_READY
;
2169 ldcp
->status
= LDC_READY
;
2172 /* if callbacks are disabled, do not notify */
2173 if (!ldcp
->cb_enabled
)
2174 notify_client
= B_FALSE
;
2176 i_ldc_clear_intr(ldcp
, CNEX_TX_INTR
);
2177 mutex_exit(&ldcp
->tx_lock
);
2179 if (notify_client
) {
2180 ldcp
->cb_inprogress
= B_TRUE
;
2181 mutex_exit(&ldcp
->lock
);
2182 rv
= ldcp
->cb(notify_event
, ldcp
->cb_arg
);
2184 DWARN(ldcp
->id
, "i_ldc_tx_hdlr: (0x%llx) callback "
2185 "failure", ldcp
->id
);
2187 mutex_enter(&ldcp
->lock
);
2188 ldcp
->cb_inprogress
= B_FALSE
;
2191 mutex_exit(&ldcp
->lock
);
2193 D1(ldcp
->id
, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp
->id
);
2195 return (DDI_INTR_CLAIMED
);
2199 * Process the Rx HV queue.
2201 * Returns 0 if data packets were found and no errors were encountered,
2202 * otherwise returns an error. In either case, the *notify argument is
2203 * set to indicate whether or not the client callback function should
2204 * be invoked. The *event argument is set to contain the callback event.
2206 * Depending on the channel mode, packets are handled differently:
2209 * For raw mode channels, when a data packet is encountered,
2210 * processing stops and all packets are left on the queue to be removed
2211 * and processed by the ldc_read code path.
2214 * For unreliable mode, when a data packet is encountered, processing
2215 * stops, and all packets are left on the queue to be removed and
2216 * processed by the ldc_read code path. Control packets are processed
2217 * inline if they are encountered before any data packets.
2220 * For reliable mode channels, all packets on the receive queue
2221 * are processed: data packets are copied to the data queue and
2222 * control packets are processed inline. Packets are only left on
2223 * the receive queue when the data queue is full.
2226 i_ldc_rx_process_hvq(ldc_chan_t
*ldcp
, boolean_t
*notify_client
,
2227 uint64_t *notify_event
)
2230 uint64_t rx_head
, rx_tail
;
2232 uint64_t link_state
, first_fragment
= 0;
2233 boolean_t trace_length
= B_TRUE
;
2235 ASSERT(MUTEX_HELD(&ldcp
->lock
));
2236 *notify_client
= B_FALSE
;
2240 * Read packet(s) from the queue
2244 link_state
= ldcp
->link_state
;
2245 rv
= hv_ldc_rx_get_state(ldcp
->id
, &rx_head
, &rx_tail
,
2249 "i_ldc_rx_process_hvq: (0x%lx) cannot read "
2250 "queue ptrs, rv=0x%d\n", ldcp
->id
, rv
);
2251 i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
2256 * reset the channel state if the channel went down
2257 * (other side unconfigured queue) or channel was reset
2258 * (other side reconfigured its queue)
2261 if (link_state
!= ldcp
->link_state
) {
2263 switch (ldcp
->link_state
) {
2264 case LDC_CHANNEL_DOWN
:
2265 D1(ldcp
->id
, "i_ldc_rx_process_hvq: channel "
2266 "link down\n", ldcp
->id
);
2267 mutex_enter(&ldcp
->tx_lock
);
2268 i_ldc_reset(ldcp
, B_FALSE
);
2269 mutex_exit(&ldcp
->tx_lock
);
2270 *notify_client
= B_TRUE
;
2271 *notify_event
= LDC_EVT_DOWN
;
2274 case LDC_CHANNEL_UP
:
2275 D1(ldcp
->id
, "i_ldc_rx_process_hvq: "
2276 "channel link up\n", ldcp
->id
);
2278 if ((ldcp
->tstate
& ~TS_IN_RESET
) == TS_OPEN
) {
2279 *notify_client
= B_TRUE
;
2280 *notify_event
= LDC_EVT_RESET
;
2281 ldcp
->tstate
|= TS_LINK_READY
;
2282 ldcp
->status
= LDC_READY
;
2286 case LDC_CHANNEL_RESET
:
2291 D1(ldcp
->id
, "i_ldc_rx_process_hvq: channel "
2292 "link reset\n", ldcp
->id
);
2293 mutex_enter(&ldcp
->tx_lock
);
2294 i_ldc_reset(ldcp
, B_FALSE
);
2295 mutex_exit(&ldcp
->tx_lock
);
2296 *notify_client
= B_TRUE
;
2297 *notify_event
= LDC_EVT_RESET
;
2303 if (LDC_INJECT_RESET(ldcp
))
2305 if (LDC_INJECT_DRNGCLEAR(ldcp
))
2306 i_ldc_mem_inject_dring_clear(ldcp
);
2309 TRACE_RXHVQ_LENGTH(ldcp
, rx_head
, rx_tail
);
2310 trace_length
= B_FALSE
;
2313 if (rx_head
== rx_tail
) {
2314 D2(ldcp
->id
, "i_ldc_rx_process_hvq: (0x%llx) "
2315 "No packets\n", ldcp
->id
);
2319 D2(ldcp
->id
, "i_ldc_rx_process_hvq: head=0x%llx, "
2320 "tail=0x%llx\n", rx_head
, rx_tail
);
2321 DUMP_LDC_PKT(ldcp
, "i_ldc_rx_process_hvq rcd",
2322 ldcp
->rx_q_va
+ rx_head
);
2324 /* get the message */
2325 msg
= (ldc_msg_t
*)(ldcp
->rx_q_va
+ rx_head
);
2327 /* if channel is in RAW mode or data pkt, notify and return */
2328 if (ldcp
->mode
== LDC_MODE_RAW
) {
2329 *notify_client
= B_TRUE
;
2330 *notify_event
|= LDC_EVT_READ
;
2334 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_INFO
)) {
2336 /* discard packet if channel is not up */
2337 if ((ldcp
->tstate
& ~TS_IN_RESET
) != TS_UP
) {
2339 /* move the head one position */
2340 rx_head
= (rx_head
+ LDC_PACKET_SIZE
) %
2341 (ldcp
->rx_q_entries
<< LDC_PACKET_SHIFT
);
2343 if (rv
= i_ldc_set_rx_head(ldcp
, rx_head
))
2348 uint64_t dq_head
, dq_tail
;
2350 /* process only RELIABLE mode data packets */
2351 if (ldcp
->mode
!= LDC_MODE_RELIABLE
) {
2352 if ((ldcp
->tstate
& TS_IN_RESET
) == 0)
2353 *notify_client
= B_TRUE
;
2354 *notify_event
|= LDC_EVT_READ
;
2358 /* don't process packet if queue full */
2359 (void) i_ldc_dq_rx_get_state(ldcp
, &dq_head
,
2361 dq_tail
= (dq_tail
+ LDC_PACKET_SIZE
) %
2362 (ldcp
->rx_dq_entries
<< LDC_PACKET_SHIFT
);
2363 if (dq_tail
== dq_head
||
2364 LDC_INJECT_DQFULL(ldcp
)) {
2371 /* Check the sequence ID for the message received */
2372 rv
= i_ldc_check_seqid(ldcp
, msg
);
2375 DWARN(ldcp
->id
, "i_ldc_rx_process_hvq: (0x%llx) "
2376 "seqid error, q_ptrs=0x%lx,0x%lx", ldcp
->id
,
2379 /* Reset last_msg_rcd to start of message */
2380 if (first_fragment
!= 0) {
2381 ldcp
->last_msg_rcd
= first_fragment
- 1;
2386 * Send a NACK due to seqid mismatch
2388 rv
= i_ldc_send_pkt(ldcp
, msg
->type
, LDC_NACK
,
2389 (msg
->ctrl
& LDC_CTRL_MASK
));
2392 cmn_err(CE_NOTE
, "i_ldc_rx_process_hvq: "
2393 "(0x%lx) err sending CTRL/DATA NACK msg\n",
2396 /* if cannot send NACK - reset channel */
2397 mutex_enter(&ldcp
->tx_lock
);
2398 i_ldc_reset(ldcp
, B_TRUE
);
2399 mutex_exit(&ldcp
->tx_lock
);
2401 *notify_client
= B_TRUE
;
2402 *notify_event
= LDC_EVT_RESET
;
2406 /* purge receive queue */
2407 (void) i_ldc_set_rx_head(ldcp
, rx_tail
);
2411 /* record the message ID */
2412 ldcp
->last_msg_rcd
= msg
->seqid
;
2414 /* process control messages */
2415 if (msg
->type
& LDC_CTRL
) {
2416 /* save current internal state */
2417 uint64_t tstate
= ldcp
->tstate
;
2419 rv
= i_ldc_ctrlmsg(ldcp
, msg
);
2421 /* re-process pkt - state was adjusted */
2424 if (rv
== ECONNRESET
) {
2425 *notify_client
= B_TRUE
;
2426 *notify_event
= LDC_EVT_RESET
;
2431 * control message processing was successful
2432 * channel transitioned to ready for communication
2434 if (rv
== 0 && ldcp
->tstate
== TS_UP
&&
2435 (tstate
& ~TS_IN_RESET
) !=
2436 (ldcp
->tstate
& ~TS_IN_RESET
)) {
2437 *notify_client
= B_TRUE
;
2438 *notify_event
= LDC_EVT_UP
;
2442 /* process data NACKs */
2443 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_NACK
)) {
2445 "i_ldc_rx_process_hvq: (0x%llx) received DATA/NACK",
2447 mutex_enter(&ldcp
->tx_lock
);
2448 i_ldc_reset(ldcp
, B_TRUE
);
2449 mutex_exit(&ldcp
->tx_lock
);
2450 *notify_client
= B_TRUE
;
2451 *notify_event
= LDC_EVT_RESET
;
2455 /* process data ACKs */
2456 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_ACK
)) {
2457 if (rv
= i_ldc_process_data_ACK(ldcp
, msg
)) {
2458 *notify_client
= B_TRUE
;
2459 *notify_event
= LDC_EVT_RESET
;
2464 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_INFO
)) {
2465 ASSERT(ldcp
->mode
== LDC_MODE_RELIABLE
);
2468 * Copy the data packet to the data queue. Note
2469 * that the copy routine updates the rx_head pointer.
2471 i_ldc_rxdq_copy(ldcp
, &rx_head
);
2473 if ((ldcp
->tstate
& TS_IN_RESET
) == 0)
2474 *notify_client
= B_TRUE
;
2475 *notify_event
|= LDC_EVT_READ
;
2477 rx_head
= (rx_head
+ LDC_PACKET_SIZE
) %
2478 (ldcp
->rx_q_entries
<< LDC_PACKET_SHIFT
);
2481 /* move the head one position */
2482 if (rv
= i_ldc_set_rx_head(ldcp
, rx_head
)) {
2483 *notify_client
= B_TRUE
;
2484 *notify_event
= LDC_EVT_RESET
;
2492 if (ldcp
->mode
== LDC_MODE_RELIABLE
) {
2493 /* ACK data packets */
2494 if ((*notify_event
&
2495 (LDC_EVT_READ
| LDC_EVT_RESET
)) == LDC_EVT_READ
) {
2497 ack_rv
= i_ldc_send_pkt(ldcp
, LDC_DATA
, LDC_ACK
, 0);
2498 if (ack_rv
&& ack_rv
!= EWOULDBLOCK
) {
2500 "i_ldc_rx_process_hvq: (0x%lx) cannot "
2501 "send ACK\n", ldcp
->id
);
2503 mutex_enter(&ldcp
->tx_lock
);
2504 i_ldc_reset(ldcp
, B_FALSE
);
2505 mutex_exit(&ldcp
->tx_lock
);
2507 *notify_client
= B_TRUE
;
2508 *notify_event
= LDC_EVT_RESET
;
2514 * If we have no more space on the data queue, make sure
2515 * there are no ACKs on the rx queue waiting to be processed.
2518 if (i_ldc_rx_ackpeek(ldcp
, rx_head
, rx_tail
) != 0) {
2519 ldcp
->rx_ack_head
= ACKPEEK_HEAD_INVALID
;
2520 *notify_client
= B_TRUE
;
2521 *notify_event
= LDC_EVT_RESET
;
2525 ldcp
->rx_ack_head
= ACKPEEK_HEAD_INVALID
;
2531 /* Return, indicating whether or not data packets were found */
2532 if ((*notify_event
& (LDC_EVT_READ
| LDC_EVT_RESET
)) == LDC_EVT_READ
)
2539 * Process any ACK packets on the HV receive queue.
2541 * This function is only used by RELIABLE mode channels when the
2542 * secondary data queue fills up and there are packets remaining on
2543 * the HV receive queue.
2546 i_ldc_rx_ackpeek(ldc_chan_t
*ldcp
, uint64_t rx_head
, uint64_t rx_tail
)
2551 if (ldcp
->rx_ack_head
== ACKPEEK_HEAD_INVALID
)
2552 ldcp
->rx_ack_head
= rx_head
;
2554 while (ldcp
->rx_ack_head
!= rx_tail
) {
2555 msg
= (ldc_msg_t
*)(ldcp
->rx_q_va
+ ldcp
->rx_ack_head
);
2557 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_ACK
)) {
2558 if (rv
= i_ldc_process_data_ACK(ldcp
, msg
))
2560 msg
->stype
&= ~LDC_ACK
;
2564 (ldcp
->rx_ack_head
+ LDC_PACKET_SIZE
) %
2565 (ldcp
->rx_q_entries
<< LDC_PACKET_SHIFT
);
2570 /* -------------------------------------------------------------------------- */
2577 * Initialize the channel. Allocate internal structure and memory for
2578 * TX/RX queues, and initialize locks.
2581 ldc_init(uint64_t id
, ldc_attr_t
*attr
, ldc_handle_t
*handle
)
2585 uint64_t ra_base
, nentries
;
2588 exit_val
= EINVAL
; /* guarantee an error if exit on failure */
2591 DWARN(id
, "ldc_init: (0x%llx) invalid attr\n", id
);
2594 if (handle
== NULL
) {
2595 DWARN(id
, "ldc_init: (0x%llx) invalid handle\n", id
);
2599 /* check if channel is valid */
2600 rv
= hv_ldc_tx_qinfo(id
, &ra_base
, &nentries
);
2601 if (rv
== H_ECHANNEL
) {
2602 DWARN(id
, "ldc_init: (0x%llx) invalid channel id\n", id
);
2606 /* check if the channel has already been initialized */
2607 mutex_enter(&ldcssp
->lock
);
2608 ldcp
= ldcssp
->chan_list
;
2609 while (ldcp
!= NULL
) {
2610 if (ldcp
->id
== id
) {
2611 DWARN(id
, "ldc_init: (0x%llx) already initialized\n",
2613 mutex_exit(&ldcssp
->lock
);
2614 return (EADDRINUSE
);
2618 mutex_exit(&ldcssp
->lock
);
2620 ASSERT(ldcp
== NULL
);
2624 /* Allocate an ldcp structure */
2625 ldcp
= kmem_zalloc(sizeof (ldc_chan_t
), KM_SLEEP
);
2628 * Initialize the channel and Tx lock
2630 * The channel 'lock' protects the entire channel and
2631 * should be acquired before initializing, resetting,
2632 * destroying or reading from a channel.
2634 * The 'tx_lock' should be acquired prior to transmitting
2635 * data over the channel. The lock should also be acquired
2636 * prior to channel reconfiguration (in order to prevent
2637 * concurrent writes).
2639 * ORDERING: When both locks are being acquired, to prevent
2640 * deadlocks, the channel lock should be always acquired prior
2643 mutex_init(&ldcp
->lock
, NULL
, MUTEX_DRIVER
, NULL
);
2644 mutex_init(&ldcp
->tx_lock
, NULL
, MUTEX_DRIVER
, NULL
);
2646 /* Initialize the channel */
2649 ldcp
->cb_arg
= NULL
;
2650 ldcp
->cb_inprogress
= B_FALSE
;
2651 ldcp
->cb_enabled
= B_FALSE
;
2654 /* Read attributes */
2655 ldcp
->mode
= attr
->mode
;
2656 ldcp
->devclass
= attr
->devclass
;
2657 ldcp
->devinst
= attr
->instance
;
2658 ldcp
->mtu
= (attr
->mtu
> 0) ? attr
->mtu
: LDC_DEFAULT_MTU
;
2661 "ldc_init: (0x%llx) channel attributes, class=0x%x, "
2662 "instance=0x%llx, mode=%d, mtu=%d\n",
2663 ldcp
->id
, ldcp
->devclass
, ldcp
->devinst
, ldcp
->mode
, ldcp
->mtu
);
2665 ldcp
->next_vidx
= 0;
2666 ldcp
->tstate
= TS_IN_RESET
;
2668 ldcp
->last_msg_snt
= LDC_INIT_SEQID
;
2669 ldcp
->last_ack_rcd
= 0;
2670 ldcp
->last_msg_rcd
= 0;
2671 ldcp
->rx_ack_head
= ACKPEEK_HEAD_INVALID
;
2673 ldcp
->stream_bufferp
= NULL
;
2674 ldcp
->exp_dring_list
= NULL
;
2675 ldcp
->imp_dring_list
= NULL
;
2676 ldcp
->mhdl_list
= NULL
;
2678 ldcp
->tx_intr_state
= LDC_INTR_NONE
;
2679 ldcp
->rx_intr_state
= LDC_INTR_NONE
;
2681 /* Initialize payload size depending on whether channel is reliable */
2682 switch (ldcp
->mode
) {
2684 ldcp
->pkt_payload
= LDC_PAYLOAD_SIZE_RAW
;
2685 ldcp
->read_p
= i_ldc_read_raw
;
2686 ldcp
->write_p
= i_ldc_write_raw
;
2688 case LDC_MODE_UNRELIABLE
:
2689 ldcp
->pkt_payload
= LDC_PAYLOAD_SIZE_UNRELIABLE
;
2690 ldcp
->read_p
= i_ldc_read_packet
;
2691 ldcp
->write_p
= i_ldc_write_packet
;
2693 case LDC_MODE_RELIABLE
:
2694 ldcp
->pkt_payload
= LDC_PAYLOAD_SIZE_RELIABLE
;
2696 ldcp
->stream_remains
= 0;
2697 ldcp
->stream_offset
= 0;
2698 ldcp
->stream_bufferp
= kmem_alloc(ldcp
->mtu
, KM_SLEEP
);
2699 ldcp
->read_p
= i_ldc_read_stream
;
2700 ldcp
->write_p
= i_ldc_write_stream
;
2704 goto cleanup_on_exit
;
2708 * qlen is (mtu * ldc_mtu_msgs) / pkt_payload. If this
2709 * value is smaller than default length of ldc_queue_entries,
2710 * qlen is set to ldc_queue_entries. Ensure that computed
2711 * length is a power-of-two value.
2713 qlen
= (ldcp
->mtu
* ldc_mtu_msgs
) / ldcp
->pkt_payload
;
2717 qlen
>>= 1; tmp
<<= 1;
2722 ldcp
->rx_q_entries
=
2723 (qlen
< ldc_queue_entries
) ? ldc_queue_entries
: qlen
;
2724 ldcp
->tx_q_entries
= ldcp
->rx_q_entries
;
2726 D1(ldcp
->id
, "ldc_init: queue length = 0x%llx\n", ldcp
->rx_q_entries
);
2728 /* Create a transmit queue */
2729 ldcp
->tx_q_va
= (uint64_t)
2730 contig_mem_alloc(ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
2731 if (ldcp
->tx_q_va
== NULL
) {
2733 "ldc_init: (0x%lx) TX queue allocation failed\n",
2736 goto cleanup_on_exit
;
2738 ldcp
->tx_q_ra
= va_to_pa((caddr_t
)ldcp
->tx_q_va
);
2740 D2(ldcp
->id
, "ldc_init: txq_va=0x%llx, txq_ra=0x%llx, entries=0x%llx\n",
2741 ldcp
->tx_q_va
, ldcp
->tx_q_ra
, ldcp
->tx_q_entries
);
2743 ldcp
->tstate
|= TS_TXQ_RDY
;
2745 /* Create a receive queue */
2746 ldcp
->rx_q_va
= (uint64_t)
2747 contig_mem_alloc(ldcp
->rx_q_entries
<< LDC_PACKET_SHIFT
);
2748 if (ldcp
->rx_q_va
== NULL
) {
2750 "ldc_init: (0x%lx) RX queue allocation failed\n",
2753 goto cleanup_on_exit
;
2755 ldcp
->rx_q_ra
= va_to_pa((caddr_t
)ldcp
->rx_q_va
);
2757 D2(ldcp
->id
, "ldc_init: rxq_va=0x%llx, rxq_ra=0x%llx, entries=0x%llx\n",
2758 ldcp
->rx_q_va
, ldcp
->rx_q_ra
, ldcp
->rx_q_entries
);
2760 ldcp
->tstate
|= TS_RXQ_RDY
;
2762 /* Setup a separate read data queue */
2763 if (ldcp
->mode
== LDC_MODE_RELIABLE
) {
2764 ldcp
->readq_get_state
= i_ldc_dq_rx_get_state
;
2765 ldcp
->readq_set_head
= i_ldc_set_rxdq_head
;
2767 /* Make sure the data queue multiplier is a power of 2 */
2768 if (!ISP2(ldc_rxdq_multiplier
)) {
2769 D1(ldcp
->id
, "ldc_init: (0x%llx) ldc_rxdq_multiplier "
2770 "not a power of 2, resetting", ldcp
->id
);
2771 ldc_rxdq_multiplier
= LDC_RXDQ_MULTIPLIER
;
2774 ldcp
->rx_dq_entries
= ldc_rxdq_multiplier
* ldcp
->rx_q_entries
;
2775 ldcp
->rx_dq_va
= (uint64_t)
2776 kmem_alloc(ldcp
->rx_dq_entries
<< LDC_PACKET_SHIFT
,
2778 if (ldcp
->rx_dq_va
== NULL
) {
2780 "ldc_init: (0x%lx) RX data queue "
2781 "allocation failed\n", ldcp
->id
);
2783 goto cleanup_on_exit
;
2786 ldcp
->rx_dq_head
= ldcp
->rx_dq_tail
= 0;
2788 D2(ldcp
->id
, "ldc_init: rx_dq_va=0x%llx, "
2789 "rx_dq_entries=0x%llx\n", ldcp
->rx_dq_va
,
2790 ldcp
->rx_dq_entries
);
2792 ldcp
->readq_get_state
= i_ldc_hvq_rx_get_state
;
2793 ldcp
->readq_set_head
= i_ldc_set_rx_head
;
2796 /* Init descriptor ring and memory handle list lock */
2797 mutex_init(&ldcp
->exp_dlist_lock
, NULL
, MUTEX_DRIVER
, NULL
);
2798 mutex_init(&ldcp
->imp_dlist_lock
, NULL
, MUTEX_DRIVER
, NULL
);
2799 mutex_init(&ldcp
->mlist_lock
, NULL
, MUTEX_DRIVER
, NULL
);
2801 /* mark status as INITialized */
2802 ldcp
->status
= LDC_INIT
;
2804 /* Add to channel list */
2805 mutex_enter(&ldcssp
->lock
);
2806 ldcp
->next
= ldcssp
->chan_list
;
2807 ldcssp
->chan_list
= ldcp
;
2808 ldcssp
->channel_count
++;
2809 mutex_exit(&ldcssp
->lock
);
2811 /* set the handle */
2812 *handle
= (ldc_handle_t
)ldcp
;
2814 D1(ldcp
->id
, "ldc_init: (0x%llx) channel initialized\n", ldcp
->id
);
2820 if (ldcp
->mode
== LDC_MODE_RELIABLE
&& ldcp
->stream_bufferp
)
2821 kmem_free(ldcp
->stream_bufferp
, ldcp
->mtu
);
2823 if (ldcp
->tstate
& TS_TXQ_RDY
)
2824 contig_mem_free((caddr_t
)ldcp
->tx_q_va
,
2825 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
));
2827 if (ldcp
->tstate
& TS_RXQ_RDY
)
2828 contig_mem_free((caddr_t
)ldcp
->rx_q_va
,
2829 (ldcp
->rx_q_entries
<< LDC_PACKET_SHIFT
));
2831 mutex_destroy(&ldcp
->tx_lock
);
2832 mutex_destroy(&ldcp
->lock
);
2835 kmem_free(ldcp
, sizeof (ldc_chan_t
));
2841 * Finalizes the LDC connection. It will return EBUSY if the
2842 * channel is open. A ldc_close() has to be done prior to
2843 * a ldc_fini operation. It frees TX/RX queues, associated
2847 ldc_fini(ldc_handle_t handle
)
2850 ldc_chan_t
*tmp_ldcp
;
2853 if (handle
== NULL
) {
2854 DWARN(DBG_ALL_LDCS
, "ldc_fini: invalid channel handle\n");
2857 ldcp
= (ldc_chan_t
*)handle
;
2860 mutex_enter(&ldcp
->lock
);
2862 if ((ldcp
->tstate
& ~TS_IN_RESET
) > TS_INIT
) {
2863 DWARN(ldcp
->id
, "ldc_fini: (0x%llx) channel is open\n",
2865 mutex_exit(&ldcp
->lock
);
2869 /* Remove from the channel list */
2870 mutex_enter(&ldcssp
->lock
);
2871 tmp_ldcp
= ldcssp
->chan_list
;
2872 if (tmp_ldcp
== ldcp
) {
2873 ldcssp
->chan_list
= ldcp
->next
;
2876 while (tmp_ldcp
!= NULL
) {
2877 if (tmp_ldcp
->next
== ldcp
) {
2878 tmp_ldcp
->next
= ldcp
->next
;
2882 tmp_ldcp
= tmp_ldcp
->next
;
2884 if (tmp_ldcp
== NULL
) {
2885 DWARN(DBG_ALL_LDCS
, "ldc_fini: invalid channel hdl\n");
2886 mutex_exit(&ldcssp
->lock
);
2887 mutex_exit(&ldcp
->lock
);
2892 ldcssp
->channel_count
--;
2894 mutex_exit(&ldcssp
->lock
);
2896 /* Free the map table for this channel */
2898 (void) hv_ldc_set_map_table(ldcp
->id
, NULL
, NULL
);
2899 if (ldcp
->mtbl
->contigmem
)
2900 contig_mem_free(ldcp
->mtbl
->table
, ldcp
->mtbl
->size
);
2902 kmem_free(ldcp
->mtbl
->table
, ldcp
->mtbl
->size
);
2903 mutex_destroy(&ldcp
->mtbl
->lock
);
2904 kmem_free(ldcp
->mtbl
, sizeof (ldc_mtbl_t
));
2907 /* Destroy descriptor ring and memory handle list lock */
2908 mutex_destroy(&ldcp
->exp_dlist_lock
);
2909 mutex_destroy(&ldcp
->imp_dlist_lock
);
2910 mutex_destroy(&ldcp
->mlist_lock
);
2912 /* Free the stream buffer for RELIABLE_MODE */
2913 if (ldcp
->mode
== LDC_MODE_RELIABLE
&& ldcp
->stream_bufferp
)
2914 kmem_free(ldcp
->stream_bufferp
, ldcp
->mtu
);
2916 /* Free the RX queue */
2917 contig_mem_free((caddr_t
)ldcp
->rx_q_va
,
2918 (ldcp
->rx_q_entries
<< LDC_PACKET_SHIFT
));
2919 ldcp
->tstate
&= ~TS_RXQ_RDY
;
2921 /* Free the RX data queue */
2922 if (ldcp
->mode
== LDC_MODE_RELIABLE
) {
2923 kmem_free((caddr_t
)ldcp
->rx_dq_va
,
2924 (ldcp
->rx_dq_entries
<< LDC_PACKET_SHIFT
));
2927 /* Free the TX queue */
2928 contig_mem_free((caddr_t
)ldcp
->tx_q_va
,
2929 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
));
2930 ldcp
->tstate
&= ~TS_TXQ_RDY
;
2932 mutex_exit(&ldcp
->lock
);
2935 mutex_destroy(&ldcp
->tx_lock
);
2936 mutex_destroy(&ldcp
->lock
);
2938 /* free channel structure */
2939 kmem_free(ldcp
, sizeof (ldc_chan_t
));
2941 D1(id
, "ldc_fini: (0x%llx) channel finalized\n", id
);
2947 * Open the LDC channel for use. It registers the TX/RX queues
2948 * with the Hypervisor. It also specifies the interrupt number
2949 * and target CPU for this channel
2952 ldc_open(ldc_handle_t handle
)
2957 if (handle
== NULL
) {
2958 DWARN(DBG_ALL_LDCS
, "ldc_open: invalid channel handle\n");
2962 ldcp
= (ldc_chan_t
*)handle
;
2964 mutex_enter(&ldcp
->lock
);
2966 if (ldcp
->tstate
< TS_INIT
) {
2968 "ldc_open: (0x%llx) channel not initialized\n", ldcp
->id
);
2969 mutex_exit(&ldcp
->lock
);
2972 if ((ldcp
->tstate
& ~TS_IN_RESET
) >= TS_OPEN
) {
2974 "ldc_open: (0x%llx) channel is already open\n", ldcp
->id
);
2975 mutex_exit(&ldcp
->lock
);
2980 * Unregister/Register the tx queue with the hypervisor
2982 rv
= hv_ldc_tx_qconf(ldcp
->id
, NULL
, NULL
);
2985 "ldc_open: (0x%lx) channel tx queue unconf failed\n",
2987 mutex_exit(&ldcp
->lock
);
2991 rv
= hv_ldc_tx_qconf(ldcp
->id
, ldcp
->tx_q_ra
, ldcp
->tx_q_entries
);
2994 "ldc_open: (0x%lx) channel tx queue conf failed\n",
2996 mutex_exit(&ldcp
->lock
);
3000 D2(ldcp
->id
, "ldc_open: (0x%llx) registered tx queue with LDC\n",
3004 * Unregister/Register the rx queue with the hypervisor
3006 rv
= hv_ldc_rx_qconf(ldcp
->id
, NULL
, NULL
);
3009 "ldc_open: (0x%lx) channel rx queue unconf failed\n",
3011 mutex_exit(&ldcp
->lock
);
3015 rv
= hv_ldc_rx_qconf(ldcp
->id
, ldcp
->rx_q_ra
, ldcp
->rx_q_entries
);
3018 "ldc_open: (0x%lx) channel rx queue conf failed\n",
3020 mutex_exit(&ldcp
->lock
);
3024 D2(ldcp
->id
, "ldc_open: (0x%llx) registered rx queue with LDC\n",
3027 ldcp
->tstate
|= TS_QCONF_RDY
;
3029 /* Register the channel with the channel nexus */
3030 rv
= i_ldc_register_channel(ldcp
);
3031 if (rv
&& rv
!= EAGAIN
) {
3033 "ldc_open: (0x%lx) channel register failed\n", ldcp
->id
);
3034 ldcp
->tstate
&= ~TS_QCONF_RDY
;
3035 (void) hv_ldc_tx_qconf(ldcp
->id
, NULL
, NULL
);
3036 (void) hv_ldc_rx_qconf(ldcp
->id
, NULL
, NULL
);
3037 mutex_exit(&ldcp
->lock
);
3041 /* mark channel in OPEN state */
3042 ldcp
->status
= LDC_OPEN
;
3044 /* Read channel state */
3045 rv
= hv_ldc_tx_get_state(ldcp
->id
,
3046 &ldcp
->tx_head
, &ldcp
->tx_tail
, &ldcp
->link_state
);
3049 "ldc_open: (0x%lx) cannot read channel state\n",
3051 (void) i_ldc_unregister_channel(ldcp
);
3052 ldcp
->tstate
&= ~TS_QCONF_RDY
;
3053 (void) hv_ldc_tx_qconf(ldcp
->id
, NULL
, NULL
);
3054 (void) hv_ldc_rx_qconf(ldcp
->id
, NULL
, NULL
);
3055 mutex_exit(&ldcp
->lock
);
3060 * set the ACKd head to current head location for reliable
3062 ldcp
->tx_ackd_head
= ldcp
->tx_head
;
3064 /* mark channel ready if HV report link is UP (peer alloc'd Rx queue) */
3065 if (ldcp
->link_state
== LDC_CHANNEL_UP
||
3066 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
3067 ldcp
->tstate
|= TS_LINK_READY
;
3068 ldcp
->status
= LDC_READY
;
3072 * if channel is being opened in RAW mode - no handshake is needed
3073 * switch the channel READY and UP state
3075 if (ldcp
->mode
== LDC_MODE_RAW
) {
3076 ldcp
->tstate
= TS_UP
; /* set bits associated with LDC UP */
3077 ldcp
->status
= LDC_UP
;
3080 mutex_exit(&ldcp
->lock
);
3083 * Increment number of open channels
3085 mutex_enter(&ldcssp
->lock
);
3086 ldcssp
->channels_open
++;
3087 mutex_exit(&ldcssp
->lock
);
3090 "ldc_open: (0x%llx) channel (0x%p) open for use "
3091 "(tstate=0x%x, status=0x%x)\n",
3092 ldcp
->id
, ldcp
, ldcp
->tstate
, ldcp
->status
);
3098 * Close the LDC connection. It will return EBUSY if there
3099 * are memory segments or descriptor rings either bound to or
3100 * mapped over the channel
3103 ldc_close(ldc_handle_t handle
)
3106 int rv
= 0, retries
= 0;
3107 boolean_t chk_done
= B_FALSE
;
3109 if (handle
== NULL
) {
3110 DWARN(DBG_ALL_LDCS
, "ldc_close: invalid channel handle\n");
3113 ldcp
= (ldc_chan_t
*)handle
;
3115 mutex_enter(&ldcp
->lock
);
3117 /* return error if channel is not open */
3118 if ((ldcp
->tstate
& ~TS_IN_RESET
) < TS_OPEN
) {
3120 "ldc_close: (0x%llx) channel is not open\n", ldcp
->id
);
3121 mutex_exit(&ldcp
->lock
);
3125 /* if any memory handles, drings, are bound or mapped cannot close */
3126 if (ldcp
->mhdl_list
!= NULL
) {
3128 "ldc_close: (0x%llx) channel has bound memory handles\n",
3130 mutex_exit(&ldcp
->lock
);
3133 if (ldcp
->exp_dring_list
!= NULL
) {
3135 "ldc_close: (0x%llx) channel has bound descriptor rings\n",
3137 mutex_exit(&ldcp
->lock
);
3140 if (ldcp
->imp_dring_list
!= NULL
) {
3142 "ldc_close: (0x%llx) channel has mapped descriptor rings\n",
3144 mutex_exit(&ldcp
->lock
);
3148 if (ldcp
->cb_inprogress
) {
3149 DWARN(ldcp
->id
, "ldc_close: (0x%llx) callback active\n",
3151 mutex_exit(&ldcp
->lock
);
3152 return (EWOULDBLOCK
);
3155 /* Obtain Tx lock */
3156 mutex_enter(&ldcp
->tx_lock
);
3159 * Wait for pending transmits to complete i.e Tx queue to drain
3160 * if there are pending pkts - wait 1 ms and retry again
3164 rv
= hv_ldc_tx_get_state(ldcp
->id
,
3165 &ldcp
->tx_head
, &ldcp
->tx_tail
, &ldcp
->link_state
);
3168 "ldc_close: (0x%lx) cannot read qptrs\n", ldcp
->id
);
3169 mutex_exit(&ldcp
->tx_lock
);
3170 mutex_exit(&ldcp
->lock
);
3174 if (ldcp
->tx_head
== ldcp
->tx_tail
||
3175 ldcp
->link_state
!= LDC_CHANNEL_UP
) {
3181 "ldc_close: (0x%llx) Tx queue drain timeout\n",
3186 /* wait for one ms and try again */
3192 * Drain the Tx and Rx queues as we are closing the
3193 * channel. We dont care about any pending packets.
3194 * We have to also drain the queue prior to clearing
3195 * pending interrupts, otherwise the HV will trigger
3196 * an interrupt the moment the interrupt state is
3199 (void) i_ldc_txq_reconf(ldcp
);
3200 i_ldc_rxq_drain(ldcp
);
3203 * Unregister the channel with the nexus
3205 while ((rv
= i_ldc_unregister_channel(ldcp
)) != 0) {
3207 mutex_exit(&ldcp
->tx_lock
);
3208 mutex_exit(&ldcp
->lock
);
3210 /* if any error other than EAGAIN return back */
3211 if (rv
!= EAGAIN
|| retries
>= ldc_max_retries
) {
3213 "ldc_close: (0x%lx) unregister failed, %d\n",
3219 * As there could be pending interrupts we need
3220 * to wait and try again
3222 drv_usecwait(ldc_close_delay
);
3223 mutex_enter(&ldcp
->lock
);
3224 mutex_enter(&ldcp
->tx_lock
);
3228 ldcp
->tstate
&= ~TS_QCONF_RDY
;
3233 rv
= hv_ldc_tx_qconf(ldcp
->id
, NULL
, NULL
);
3236 "ldc_close: (0x%lx) channel TX queue unconf failed\n",
3238 mutex_exit(&ldcp
->tx_lock
);
3239 mutex_exit(&ldcp
->lock
);
3242 rv
= hv_ldc_rx_qconf(ldcp
->id
, NULL
, NULL
);
3245 "ldc_close: (0x%lx) channel RX queue unconf failed\n",
3247 mutex_exit(&ldcp
->tx_lock
);
3248 mutex_exit(&ldcp
->lock
);
3252 /* Reset channel state information */
3253 i_ldc_reset_state(ldcp
);
3255 /* Mark channel as down and in initialized state */
3256 ldcp
->tx_ackd_head
= 0;
3258 ldcp
->tstate
= TS_IN_RESET
|TS_INIT
;
3259 ldcp
->status
= LDC_INIT
;
3261 mutex_exit(&ldcp
->tx_lock
);
3262 mutex_exit(&ldcp
->lock
);
3264 /* Decrement number of open channels */
3265 mutex_enter(&ldcssp
->lock
);
3266 ldcssp
->channels_open
--;
3267 mutex_exit(&ldcssp
->lock
);
3269 D1(ldcp
->id
, "ldc_close: (0x%llx) channel closed\n", ldcp
->id
);
3275 * Register channel callback
3278 ldc_reg_callback(ldc_handle_t handle
,
3279 uint_t(*cb
)(uint64_t event
, caddr_t arg
), caddr_t arg
)
3283 if (handle
== NULL
) {
3285 "ldc_reg_callback: invalid channel handle\n");
3288 if (((uint64_t)cb
) < KERNELBASE
) {
3289 DWARN(DBG_ALL_LDCS
, "ldc_reg_callback: invalid callback\n");
3292 ldcp
= (ldc_chan_t
*)handle
;
3294 mutex_enter(&ldcp
->lock
);
3297 DWARN(ldcp
->id
, "ldc_reg_callback: (0x%llx) callback exists\n",
3299 mutex_exit(&ldcp
->lock
);
3302 if (ldcp
->cb_inprogress
) {
3303 DWARN(ldcp
->id
, "ldc_reg_callback: (0x%llx) callback active\n",
3305 mutex_exit(&ldcp
->lock
);
3306 return (EWOULDBLOCK
);
3311 ldcp
->cb_enabled
= B_TRUE
;
3314 "ldc_reg_callback: (0x%llx) registered callback for channel\n",
3317 mutex_exit(&ldcp
->lock
);
3323 * Unregister channel callback
3326 ldc_unreg_callback(ldc_handle_t handle
)
3330 if (handle
== NULL
) {
3332 "ldc_unreg_callback: invalid channel handle\n");
3335 ldcp
= (ldc_chan_t
*)handle
;
3337 mutex_enter(&ldcp
->lock
);
3339 if (ldcp
->cb
== NULL
) {
3341 "ldc_unreg_callback: (0x%llx) no callback exists\n",
3343 mutex_exit(&ldcp
->lock
);
3346 if (ldcp
->cb_inprogress
) {
3348 "ldc_unreg_callback: (0x%llx) callback active\n",
3350 mutex_exit(&ldcp
->lock
);
3351 return (EWOULDBLOCK
);
3355 ldcp
->cb_arg
= NULL
;
3356 ldcp
->cb_enabled
= B_FALSE
;
3359 "ldc_unreg_callback: (0x%llx) unregistered callback for channel\n",
3362 mutex_exit(&ldcp
->lock
);
3369 * Bring a channel up by initiating a handshake with the peer
3370 * This call is asynchronous. It will complete at a later point
3371 * in time when the peer responds back with an RTR.
3374 ldc_up(ldc_handle_t handle
)
3379 uint64_t tx_tail
, tstate
, link_state
;
3381 if (handle
== NULL
) {
3382 DWARN(DBG_ALL_LDCS
, "ldc_up: invalid channel handle\n");
3385 ldcp
= (ldc_chan_t
*)handle
;
3387 mutex_enter(&ldcp
->lock
);
3389 D1(ldcp
->id
, "ldc_up: (0x%llx) doing channel UP\n", ldcp
->id
);
3391 /* clear the reset state */
3392 tstate
= ldcp
->tstate
;
3393 ldcp
->tstate
&= ~TS_IN_RESET
;
3395 if (ldcp
->tstate
== TS_UP
) {
3397 "ldc_up: (0x%llx) channel is already in UP state\n",
3400 /* mark channel as up */
3401 ldcp
->status
= LDC_UP
;
3404 * if channel was in reset state and there was
3405 * pending data clear interrupt state. this will
3406 * trigger an interrupt, causing the RX handler to
3407 * to invoke the client's callback
3409 if ((tstate
& TS_IN_RESET
) &&
3410 ldcp
->rx_intr_state
== LDC_INTR_PEND
) {
3412 "ldc_up: (0x%llx) channel has pending data, "
3413 "clearing interrupt\n", ldcp
->id
);
3414 i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
3417 mutex_exit(&ldcp
->lock
);
3421 /* if the channel is in RAW mode - mark it as UP, if READY */
3422 if (ldcp
->mode
== LDC_MODE_RAW
&& ldcp
->tstate
>= TS_READY
) {
3423 ldcp
->tstate
= TS_UP
;
3424 mutex_exit(&ldcp
->lock
);
3428 /* Don't start another handshake if there is one in progress */
3431 "ldc_up: (0x%llx) channel handshake in progress\n",
3433 mutex_exit(&ldcp
->lock
);
3437 mutex_enter(&ldcp
->tx_lock
);
3439 /* save current link state */
3440 link_state
= ldcp
->link_state
;
3442 /* get the current tail for the LDC msg */
3443 rv
= i_ldc_get_tx_tail(ldcp
, &tx_tail
);
3445 D1(ldcp
->id
, "ldc_up: (0x%llx) cannot initiate handshake\n",
3447 mutex_exit(&ldcp
->tx_lock
);
3448 mutex_exit(&ldcp
->lock
);
3449 return (ECONNREFUSED
);
3453 * If i_ldc_get_tx_tail() changed link_state to either RESET or UP,
3454 * from a previous state of DOWN, then mark the channel as
3455 * being ready for handshake.
3457 if ((link_state
== LDC_CHANNEL_DOWN
) &&
3458 (link_state
!= ldcp
->link_state
)) {
3460 ASSERT((ldcp
->link_state
== LDC_CHANNEL_RESET
) ||
3461 (ldcp
->link_state
== LDC_CHANNEL_UP
));
3463 if (ldcp
->mode
== LDC_MODE_RAW
) {
3464 ldcp
->status
= LDC_UP
;
3465 ldcp
->tstate
= TS_UP
;
3466 mutex_exit(&ldcp
->tx_lock
);
3467 mutex_exit(&ldcp
->lock
);
3470 ldcp
->status
= LDC_READY
;
3471 ldcp
->tstate
|= TS_LINK_READY
;
3476 ldcmsg
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
3479 ldcmsg
->type
= LDC_CTRL
;
3480 ldcmsg
->stype
= LDC_INFO
;
3481 ldcmsg
->ctrl
= LDC_VER
;
3482 ldcp
->next_vidx
= 0;
3483 bcopy(&ldc_versions
[0], ldcmsg
->udata
, sizeof (ldc_versions
[0]));
3485 DUMP_LDC_PKT(ldcp
, "ldc_up snd ver", (uint64_t)ldcmsg
);
3487 /* initiate the send by calling into HV and set the new tail */
3488 tx_tail
= (tx_tail
+ LDC_PACKET_SIZE
) %
3489 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
3491 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
3494 "ldc_up: (0x%llx) cannot initiate handshake rv=%d\n",
3496 mutex_exit(&ldcp
->tx_lock
);
3497 mutex_exit(&ldcp
->lock
);
3501 ldcp
->hstate
|= TS_SENT_VER
;
3502 ldcp
->tx_tail
= tx_tail
;
3503 D1(ldcp
->id
, "ldc_up: (0x%llx) channel up initiated\n", ldcp
->id
);
3505 mutex_exit(&ldcp
->tx_lock
);
3506 mutex_exit(&ldcp
->lock
);
3513 * Bring a channel down by resetting its state and queues
3516 ldc_down(ldc_handle_t handle
)
3520 if (handle
== NULL
) {
3521 DWARN(DBG_ALL_LDCS
, "ldc_down: invalid channel handle\n");
3524 ldcp
= (ldc_chan_t
*)handle
;
3525 mutex_enter(&ldcp
->lock
);
3526 mutex_enter(&ldcp
->tx_lock
);
3527 i_ldc_reset(ldcp
, B_TRUE
);
3528 mutex_exit(&ldcp
->tx_lock
);
3529 mutex_exit(&ldcp
->lock
);
3535 * Get the current channel status
3538 ldc_status(ldc_handle_t handle
, ldc_status_t
*status
)
3542 if (handle
== NULL
|| status
== NULL
) {
3543 DWARN(DBG_ALL_LDCS
, "ldc_status: invalid argument\n");
3546 ldcp
= (ldc_chan_t
*)handle
;
3548 *status
= ((ldc_chan_t
*)handle
)->status
;
3551 "ldc_status: (0x%llx) returned status %d\n", ldcp
->id
, *status
);
3557 * Set the channel's callback mode - enable/disable callbacks
3560 ldc_set_cb_mode(ldc_handle_t handle
, ldc_cb_mode_t cmode
)
3564 if (handle
== NULL
) {
3566 "ldc_set_intr_mode: invalid channel handle\n");
3569 ldcp
= (ldc_chan_t
*)handle
;
3572 * Record no callbacks should be invoked
3574 mutex_enter(&ldcp
->lock
);
3577 case LDC_CB_DISABLE
:
3578 if (!ldcp
->cb_enabled
) {
3580 "ldc_set_cb_mode: (0x%llx) callbacks disabled\n",
3584 ldcp
->cb_enabled
= B_FALSE
;
3586 D1(ldcp
->id
, "ldc_set_cb_mode: (0x%llx) disabled callbacks\n",
3591 if (ldcp
->cb_enabled
) {
3593 "ldc_set_cb_mode: (0x%llx) callbacks enabled\n",
3597 ldcp
->cb_enabled
= B_TRUE
;
3599 D1(ldcp
->id
, "ldc_set_cb_mode: (0x%llx) enabled callbacks\n",
3604 mutex_exit(&ldcp
->lock
);
3610 * Check to see if there are packets on the incoming queue
3611 * Will return hasdata = B_FALSE if there are no packets
3614 ldc_chkq(ldc_handle_t handle
, boolean_t
*hasdata
)
3617 uint64_t rx_head
, rx_tail
;
3620 if (handle
== NULL
) {
3621 DWARN(DBG_ALL_LDCS
, "ldc_chkq: invalid channel handle\n");
3624 ldcp
= (ldc_chan_t
*)handle
;
3628 mutex_enter(&ldcp
->lock
);
3630 if (ldcp
->tstate
!= TS_UP
) {
3632 "ldc_chkq: (0x%llx) channel is not up\n", ldcp
->id
);
3633 mutex_exit(&ldcp
->lock
);
3634 return (ECONNRESET
);
3637 /* Read packet(s) from the queue */
3638 rv
= hv_ldc_rx_get_state(ldcp
->id
, &rx_head
, &rx_tail
,
3642 "ldc_chkq: (0x%lx) unable to read queue ptrs", ldcp
->id
);
3643 mutex_exit(&ldcp
->lock
);
3647 /* reset the channel state if the channel went down */
3648 if (ldcp
->link_state
== LDC_CHANNEL_DOWN
||
3649 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
3650 mutex_enter(&ldcp
->tx_lock
);
3651 i_ldc_reset(ldcp
, B_FALSE
);
3652 mutex_exit(&ldcp
->tx_lock
);
3653 mutex_exit(&ldcp
->lock
);
3654 return (ECONNRESET
);
3657 switch (ldcp
->mode
) {
3660 * In raw mode, there are no ctrl packets, so checking
3661 * if the queue is non-empty is sufficient.
3663 *hasdata
= (rx_head
!= rx_tail
);
3666 case LDC_MODE_UNRELIABLE
:
3668 * In unreliable mode, if the queue is non-empty, we need
3669 * to check if it actually contains unread data packets.
3670 * The queue may just contain ctrl packets.
3672 if (rx_head
!= rx_tail
) {
3673 *hasdata
= (i_ldc_chkq(ldcp
) == 0);
3675 * If no data packets were found on the queue,
3676 * all packets must have been control packets
3677 * which will now have been processed, leaving
3678 * the queue empty. If the interrupt state
3679 * is pending, we need to clear the interrupt
3682 if (*hasdata
== B_FALSE
&&
3683 ldcp
->rx_intr_state
== LDC_INTR_PEND
) {
3684 i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
3689 case LDC_MODE_RELIABLE
:
3691 * In reliable mode, first check for 'stream_remains' > 0.
3692 * Otherwise, if the data queue head and tail pointers
3693 * differ, there must be data to read.
3695 if (ldcp
->stream_remains
> 0)
3698 *hasdata
= (ldcp
->rx_dq_head
!= ldcp
->rx_dq_tail
);
3702 cmn_err(CE_WARN
, "ldc_chkq: (0x%lx) unexpected channel mode "
3703 "(0x%x)", ldcp
->id
, ldcp
->mode
);
3704 mutex_exit(&ldcp
->lock
);
3708 mutex_exit(&ldcp
->lock
);
3715 * Read 'size' amount of bytes or less. If incoming buffer
3716 * is more than 'size', ENOBUFS is returned.
3718 * On return, size contains the number of bytes read.
3721 ldc_read(ldc_handle_t handle
, caddr_t bufp
, size_t *sizep
)
3724 uint64_t rx_head
= 0, rx_tail
= 0;
3725 int rv
= 0, exit_val
;
3727 if (handle
== NULL
) {
3728 DWARN(DBG_ALL_LDCS
, "ldc_read: invalid channel handle\n");
3732 ldcp
= (ldc_chan_t
*)handle
;
3735 mutex_enter(&ldcp
->lock
);
3737 if (ldcp
->tstate
!= TS_UP
) {
3739 "ldc_read: (0x%llx) channel is not in UP state\n",
3741 exit_val
= ECONNRESET
;
3742 } else if (ldcp
->mode
== LDC_MODE_RELIABLE
) {
3743 TRACE_RXDQ_LENGTH(ldcp
);
3744 exit_val
= ldcp
->read_p(ldcp
, bufp
, sizep
);
3747 * For reliable mode channels, the interrupt
3748 * state is only set to pending during
3749 * interrupt handling when the secondary data
3750 * queue became full, leaving unprocessed
3751 * packets on the Rx queue. If the interrupt
3752 * state is pending and space is now available
3753 * on the data queue, clear the interrupt.
3755 if (ldcp
->rx_intr_state
== LDC_INTR_PEND
&&
3756 Q_CONTIG_SPACE(ldcp
->rx_dq_head
, ldcp
->rx_dq_tail
,
3757 ldcp
->rx_dq_entries
<< LDC_PACKET_SHIFT
) >=
3759 /* data queue is not full */
3760 i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
3763 mutex_exit(&ldcp
->lock
);
3766 exit_val
= ldcp
->read_p(ldcp
, bufp
, sizep
);
3770 * if queue has been drained - clear interrupt
3772 rv
= hv_ldc_rx_get_state(ldcp
->id
, &rx_head
, &rx_tail
,
3775 cmn_err(CE_WARN
, "ldc_read: (0x%lx) unable to read queue ptrs",
3777 mutex_enter(&ldcp
->tx_lock
);
3778 i_ldc_reset(ldcp
, B_TRUE
);
3779 mutex_exit(&ldcp
->tx_lock
);
3780 mutex_exit(&ldcp
->lock
);
3781 return (ECONNRESET
);
3784 if (exit_val
== 0) {
3785 if (ldcp
->link_state
== LDC_CHANNEL_DOWN
||
3786 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
3787 mutex_enter(&ldcp
->tx_lock
);
3788 i_ldc_reset(ldcp
, B_FALSE
);
3789 exit_val
= ECONNRESET
;
3790 mutex_exit(&ldcp
->tx_lock
);
3793 (ldcp
->rx_intr_state
== LDC_INTR_PEND
) &&
3794 (rx_head
== rx_tail
)) {
3795 i_ldc_clear_intr(ldcp
, CNEX_RX_INTR
);
3799 mutex_exit(&ldcp
->lock
);
3804 * Basic raw mondo read -
3805 * no interpretation of mondo contents at all.
3807 * Enter and exit with ldcp->lock held by caller
3810 i_ldc_read_raw(ldc_chan_t
*ldcp
, caddr_t target_bufp
, size_t *sizep
)
3812 uint64_t q_size_mask
;
3816 uint64_t rx_head
, rx_tail
;
3820 if (space
< LDC_PAYLOAD_SIZE_RAW
)
3823 ASSERT(mutex_owned(&ldcp
->lock
));
3825 /* compute mask for increment */
3826 q_size_mask
= (ldcp
->rx_q_entries
-1)<<LDC_PACKET_SHIFT
;
3829 * Read packet(s) from the queue
3831 rv
= hv_ldc_rx_get_state(ldcp
->id
, &rx_head
, &rx_tail
,
3835 "ldc_read_raw: (0x%lx) unable to read queue ptrs",
3839 D1(ldcp
->id
, "ldc_read_raw: (0x%llx) rxh=0x%llx,"
3840 " rxt=0x%llx, st=0x%llx\n",
3841 ldcp
->id
, rx_head
, rx_tail
, ldcp
->link_state
);
3843 /* reset the channel state if the channel went down */
3844 if (ldcp
->link_state
== LDC_CHANNEL_DOWN
||
3845 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
3846 mutex_enter(&ldcp
->tx_lock
);
3847 i_ldc_reset(ldcp
, B_FALSE
);
3848 mutex_exit(&ldcp
->tx_lock
);
3849 return (ECONNRESET
);
3853 * Check for empty queue
3855 if (rx_head
== rx_tail
) {
3860 /* get the message */
3861 msgp
= (ldc_msg_t
*)(ldcp
->rx_q_va
+ rx_head
);
3863 /* if channel is in RAW mode, copy data and return */
3864 msgbufp
= (uint8_t *)&(msgp
->raw
[0]);
3866 bcopy(msgbufp
, target_bufp
, LDC_PAYLOAD_SIZE_RAW
);
3868 DUMP_PAYLOAD(ldcp
->id
, msgbufp
);
3870 *sizep
= LDC_PAYLOAD_SIZE_RAW
;
3872 rx_head
= (rx_head
+ LDC_PACKET_SIZE
) & q_size_mask
;
3873 rv
= i_ldc_set_rx_head(ldcp
, rx_head
);
3879 * Process LDC mondos to build larger packets
3880 * with either un-reliable or reliable delivery.
3882 * Enter and exit with ldcp->lock held by caller
3885 i_ldc_read_packet(ldc_chan_t
*ldcp
, caddr_t target_bufp
, size_t *sizep
)
3888 uint64_t rx_head
= 0, rx_tail
= 0;
3889 uint64_t curr_head
= 0;
3892 size_t len
= 0, bytes_read
= 0;
3894 uint64_t q_va
, q_size_mask
;
3895 uint64_t first_fragment
= 0;
3897 target
= target_bufp
;
3899 ASSERT(mutex_owned(&ldcp
->lock
));
3901 /* check if the buffer and size are valid */
3902 if (target_bufp
== NULL
|| *sizep
== 0) {
3903 DWARN(ldcp
->id
, "ldc_read: (0x%llx) invalid buffer/size\n",
3908 /* Set q_va and compute increment mask for the appropriate queue */
3909 if (ldcp
->mode
== LDC_MODE_RELIABLE
) {
3910 q_va
= ldcp
->rx_dq_va
;
3911 q_size_mask
= (ldcp
->rx_dq_entries
-1)<<LDC_PACKET_SHIFT
;
3913 q_va
= ldcp
->rx_q_va
;
3914 q_size_mask
= (ldcp
->rx_q_entries
-1)<<LDC_PACKET_SHIFT
;
3918 * Read packet(s) from the queue
3920 rv
= ldcp
->readq_get_state(ldcp
, &curr_head
, &rx_tail
,
3923 cmn_err(CE_WARN
, "ldc_read: (0x%lx) unable to read queue ptrs",
3925 mutex_enter(&ldcp
->tx_lock
);
3926 i_ldc_reset(ldcp
, B_TRUE
);
3927 mutex_exit(&ldcp
->tx_lock
);
3928 return (ECONNRESET
);
3930 D1(ldcp
->id
, "ldc_read: (0x%llx) chd=0x%llx, tl=0x%llx, st=0x%llx\n",
3931 ldcp
->id
, curr_head
, rx_tail
, ldcp
->link_state
);
3933 /* reset the channel state if the channel went down */
3934 if (ldcp
->link_state
!= LDC_CHANNEL_UP
)
3935 goto channel_is_reset
;
3939 if (curr_head
== rx_tail
) {
3941 * If a data queue is being used, check the Rx HV
3942 * queue. This will copy over any new data packets
3943 * that have arrived.
3945 if (ldcp
->mode
== LDC_MODE_RELIABLE
)
3946 (void) i_ldc_chkq(ldcp
);
3948 rv
= ldcp
->readq_get_state(ldcp
,
3949 &rx_head
, &rx_tail
, &ldcp
->link_state
);
3952 "ldc_read: (0x%lx) cannot read queue ptrs",
3954 mutex_enter(&ldcp
->tx_lock
);
3955 i_ldc_reset(ldcp
, B_TRUE
);
3956 mutex_exit(&ldcp
->tx_lock
);
3957 return (ECONNRESET
);
3960 if (ldcp
->link_state
!= LDC_CHANNEL_UP
)
3961 goto channel_is_reset
;
3963 if (curr_head
== rx_tail
) {
3965 /* If in the middle of a fragmented xfer */
3966 if (first_fragment
!= 0) {
3968 /* wait for ldc_delay usecs */
3969 drv_usecwait(ldc_delay
);
3971 if (++retries
< ldc_max_retries
)
3975 if (ldcp
->mode
!= LDC_MODE_RELIABLE
)
3976 ldcp
->last_msg_rcd
=
3978 DWARN(DBG_ALL_LDCS
, "ldc_read: "
3979 "(0x%llx) read timeout", ldcp
->id
);
3989 "ldc_read: (0x%llx) chd=0x%llx, rxhd=0x%llx, rxtl=0x%llx\n",
3990 ldcp
->id
, curr_head
, rx_head
, rx_tail
);
3992 /* get the message */
3993 msg
= (ldc_msg_t
*)(q_va
+ curr_head
);
3995 DUMP_LDC_PKT(ldcp
, "ldc_read received pkt",
3996 ldcp
->rx_q_va
+ curr_head
);
3998 /* Check the message ID for the message received */
3999 if (ldcp
->mode
!= LDC_MODE_RELIABLE
) {
4000 if ((rv
= i_ldc_check_seqid(ldcp
, msg
)) != 0) {
4002 DWARN(ldcp
->id
, "ldc_read: (0x%llx) seqid "
4003 "error, q_ptrs=0x%lx,0x%lx",
4004 ldcp
->id
, rx_head
, rx_tail
);
4006 /* throw away data */
4009 /* Reset last_msg_rcd to start of message */
4010 if (first_fragment
!= 0) {
4011 ldcp
->last_msg_rcd
= first_fragment
- 1;
4015 * Send a NACK -- invalid seqid
4016 * get the current tail for the response
4018 rv
= i_ldc_send_pkt(ldcp
, msg
->type
, LDC_NACK
,
4019 (msg
->ctrl
& LDC_CTRL_MASK
));
4022 "ldc_read: (0x%lx) err sending "
4023 "NACK msg\n", ldcp
->id
);
4025 /* if cannot send NACK - reset chan */
4026 mutex_enter(&ldcp
->tx_lock
);
4027 i_ldc_reset(ldcp
, B_FALSE
);
4028 mutex_exit(&ldcp
->tx_lock
);
4033 /* purge receive queue */
4034 rv
= i_ldc_set_rx_head(ldcp
, rx_tail
);
4040 * Process any messages of type CTRL messages
4041 * Future implementations should try to pass these
4042 * to LDC link by resetting the intr state.
4044 * NOTE: not done as a switch() as type can be
4047 if (msg
->type
& LDC_CTRL
) {
4048 if (rv
= i_ldc_ctrlmsg(ldcp
, msg
)) {
4051 rv
= i_ldc_set_rx_head(ldcp
, rx_tail
);
4058 /* process data ACKs */
4059 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_ACK
)) {
4060 if (rv
= i_ldc_process_data_ACK(ldcp
, msg
)) {
4067 /* process data NACKs */
4068 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_NACK
)) {
4070 "ldc_read: (0x%llx) received DATA/NACK",
4072 mutex_enter(&ldcp
->tx_lock
);
4073 i_ldc_reset(ldcp
, B_TRUE
);
4074 mutex_exit(&ldcp
->tx_lock
);
4075 return (ECONNRESET
);
4079 /* process data messages */
4080 if ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_INFO
)) {
4082 uint8_t *msgbuf
= (uint8_t *)(
4083 (ldcp
->mode
== LDC_MODE_RELIABLE
) ?
4084 msg
->rdata
: msg
->udata
);
4087 "ldc_read: (0x%llx) received data msg\n", ldcp
->id
);
4089 /* get the packet length */
4090 len
= (msg
->env
& LDC_LEN_MASK
);
4093 * FUTURE OPTIMIZATION:
4094 * dont need to set q head for every
4095 * packet we read just need to do this when
4096 * we are done or need to wait for more
4097 * mondos to make a full packet - this is
4098 * currently expensive.
4101 if (first_fragment
== 0) {
4104 * first packets should always have the start
4105 * bit set (even for a single packet). If not
4106 * throw away the packet
4108 if (!(msg
->env
& LDC_FRAG_START
)) {
4111 "ldc_read: (0x%llx) not start - "
4112 "frag=%x\n", ldcp
->id
,
4113 (msg
->env
) & LDC_FRAG_MASK
);
4115 /* toss pkt, inc head, cont reading */
4117 target
= target_bufp
;
4119 (curr_head
+ LDC_PACKET_SIZE
)
4121 if (rv
= ldcp
->readq_set_head(ldcp
,
4128 first_fragment
= msg
->seqid
;
4130 /* check to see if this is a pkt w/ START bit */
4131 if (msg
->env
& LDC_FRAG_START
) {
4133 "ldc_read:(0x%llx) unexpected pkt"
4134 " env=0x%x discarding %d bytes,"
4135 " lastmsg=%d, currentmsg=%d\n",
4136 ldcp
->id
, msg
->env
&LDC_FRAG_MASK
,
4137 bytes_read
, ldcp
->last_msg_rcd
,
4140 /* throw data we have read so far */
4142 target
= target_bufp
;
4143 first_fragment
= msg
->seqid
;
4145 if (rv
= ldcp
->readq_set_head(ldcp
,
4151 /* copy (next) pkt into buffer */
4152 if (len
<= (*sizep
- bytes_read
)) {
4153 bcopy(msgbuf
, target
, len
);
4158 * there is not enough space in the buffer to
4159 * read this pkt. throw message away & continue
4160 * reading data from queue
4163 "ldc_read: (0x%llx) buffer too small, "
4164 "head=0x%lx, expect=%d, got=%d\n", ldcp
->id
,
4165 curr_head
, *sizep
, bytes_read
+len
);
4168 target
= target_bufp
;
4171 /* throw away everything received so far */
4172 if (rv
= ldcp
->readq_set_head(ldcp
, curr_head
))
4175 /* continue reading remaining pkts */
4180 /* set the message id */
4181 if (ldcp
->mode
!= LDC_MODE_RELIABLE
)
4182 ldcp
->last_msg_rcd
= msg
->seqid
;
4184 /* move the head one position */
4185 curr_head
= (curr_head
+ LDC_PACKET_SIZE
) & q_size_mask
;
4187 if (msg
->env
& LDC_FRAG_STOP
) {
4190 * All pkts that are part of this fragmented transfer
4191 * have been read or this was a single pkt read
4192 * or there was an error
4195 /* set the queue head */
4196 if (rv
= ldcp
->readq_set_head(ldcp
, curr_head
))
4199 *sizep
= bytes_read
;
4204 /* advance head if it is a CTRL packet or a DATA ACK packet */
4205 if ((msg
->type
& LDC_CTRL
) ||
4206 ((msg
->type
& LDC_DATA
) && (msg
->stype
& LDC_ACK
))) {
4208 /* set the queue head */
4209 if (rv
= ldcp
->readq_set_head(ldcp
, curr_head
)) {
4214 D2(ldcp
->id
, "ldc_read: (0x%llx) set ACK qhead 0x%llx",
4215 ldcp
->id
, curr_head
);
4220 D2(ldcp
->id
, "ldc_read: (0x%llx) end size=%d", ldcp
->id
, *sizep
);
4225 mutex_enter(&ldcp
->tx_lock
);
4226 i_ldc_reset(ldcp
, B_FALSE
);
4227 mutex_exit(&ldcp
->tx_lock
);
4228 return (ECONNRESET
);
4232 * Fetch and buffer incoming packets so we can hand them back as
4233 * a basic byte stream.
4235 * Enter and exit with ldcp->lock held by caller
4238 i_ldc_read_stream(ldc_chan_t
*ldcp
, caddr_t target_bufp
, size_t *sizep
)
4243 ASSERT(mutex_owned(&ldcp
->lock
));
4245 D2(ldcp
->id
, "i_ldc_read_stream: (0x%llx) buffer size=%d",
4248 if (ldcp
->stream_remains
== 0) {
4250 rv
= i_ldc_read_packet(ldcp
,
4251 (caddr_t
)ldcp
->stream_bufferp
, &size
);
4252 D2(ldcp
->id
, "i_ldc_read_stream: read packet (0x%llx) size=%d",
4258 ldcp
->stream_remains
= size
;
4259 ldcp
->stream_offset
= 0;
4262 size
= MIN(ldcp
->stream_remains
, *sizep
);
4264 bcopy(ldcp
->stream_bufferp
+ ldcp
->stream_offset
, target_bufp
, size
);
4265 ldcp
->stream_offset
+= size
;
4266 ldcp
->stream_remains
-= size
;
4268 D2(ldcp
->id
, "i_ldc_read_stream: (0x%llx) fill from buffer size=%d",
4276 * Write specified amount of bytes to the channel
4277 * in multiple pkts of pkt_payload size. Each
4278 * packet is tagged with an unique packet ID in
4279 * the case of a reliable link.
4281 * On return, size contains the number of bytes written.
4284 ldc_write(ldc_handle_t handle
, caddr_t buf
, size_t *sizep
)
4289 if (handle
== NULL
) {
4290 DWARN(DBG_ALL_LDCS
, "ldc_write: invalid channel handle\n");
4293 ldcp
= (ldc_chan_t
*)handle
;
4295 mutex_enter(&ldcp
->tx_lock
);
4297 /* check if non-zero data to write */
4298 if (buf
== NULL
|| sizep
== NULL
) {
4299 DWARN(ldcp
->id
, "ldc_write: (0x%llx) invalid data write\n",
4301 mutex_exit(&ldcp
->tx_lock
);
4306 DWARN(ldcp
->id
, "ldc_write: (0x%llx) write size of zero\n",
4308 mutex_exit(&ldcp
->tx_lock
);
4312 /* Check if channel is UP for data exchange */
4313 if (ldcp
->tstate
!= TS_UP
) {
4315 "ldc_write: (0x%llx) channel is not in UP state\n",
4320 rv
= ldcp
->write_p(ldcp
, buf
, sizep
);
4323 mutex_exit(&ldcp
->tx_lock
);
4329 * Write a raw packet to the channel
4330 * On return, size contains the number of bytes written.
4333 i_ldc_write_raw(ldc_chan_t
*ldcp
, caddr_t buf
, size_t *sizep
)
4336 uint64_t tx_head
, tx_tail
, new_tail
;
4340 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
4341 ASSERT(ldcp
->mode
== LDC_MODE_RAW
);
4346 * Check to see if the packet size is less than or
4347 * equal to packet size support in raw mode
4349 if (size
> ldcp
->pkt_payload
) {
4351 "ldc_write: (0x%llx) invalid size (0x%llx) for RAW mode\n",
4357 /* get the qptrs for the tx queue */
4358 rv
= hv_ldc_tx_get_state(ldcp
->id
,
4359 &ldcp
->tx_head
, &ldcp
->tx_tail
, &ldcp
->link_state
);
4362 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp
->id
);
4367 if (ldcp
->link_state
== LDC_CHANNEL_DOWN
||
4368 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
4370 "ldc_write: (0x%llx) channel down/reset\n", ldcp
->id
);
4373 if (mutex_tryenter(&ldcp
->lock
)) {
4374 i_ldc_reset(ldcp
, B_FALSE
);
4375 mutex_exit(&ldcp
->lock
);
4378 * Release Tx lock, and then reacquire channel
4379 * and Tx lock in correct order
4381 mutex_exit(&ldcp
->tx_lock
);
4382 mutex_enter(&ldcp
->lock
);
4383 mutex_enter(&ldcp
->tx_lock
);
4384 i_ldc_reset(ldcp
, B_FALSE
);
4385 mutex_exit(&ldcp
->lock
);
4387 return (ECONNRESET
);
4390 tx_tail
= ldcp
->tx_tail
;
4391 tx_head
= ldcp
->tx_head
;
4392 new_tail
= (tx_tail
+ LDC_PACKET_SIZE
) &
4393 ((ldcp
->tx_q_entries
-1) << LDC_PACKET_SHIFT
);
4395 if (new_tail
== tx_head
) {
4397 "ldc_write: (0x%llx) TX queue is full\n", ldcp
->id
);
4399 return (EWOULDBLOCK
);
4402 D2(ldcp
->id
, "ldc_write: (0x%llx) start xfer size=%d",
4405 /* Send the data now */
4406 ldcmsg
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
4408 /* copy the data into pkt */
4409 bcopy((uint8_t *)buf
, ldcmsg
, size
);
4411 /* increment tail */
4415 * All packets have been copied into the TX queue
4416 * update the tail ptr in the HV
4418 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
4420 if (rv
== EWOULDBLOCK
) {
4421 DWARN(ldcp
->id
, "ldc_write: (0x%llx) write timed out\n",
4424 return (EWOULDBLOCK
);
4428 if (mutex_tryenter(&ldcp
->lock
)) {
4429 i_ldc_reset(ldcp
, B_FALSE
);
4430 mutex_exit(&ldcp
->lock
);
4433 * Release Tx lock, and then reacquire channel
4434 * and Tx lock in correct order
4436 mutex_exit(&ldcp
->tx_lock
);
4437 mutex_enter(&ldcp
->lock
);
4438 mutex_enter(&ldcp
->tx_lock
);
4439 i_ldc_reset(ldcp
, B_FALSE
);
4440 mutex_exit(&ldcp
->lock
);
4442 return (ECONNRESET
);
4445 ldcp
->tx_tail
= tx_tail
;
4448 D2(ldcp
->id
, "ldc_write: (0x%llx) end xfer size=%d", ldcp
->id
, size
);
4455 * Write specified amount of bytes to the channel
4456 * in multiple pkts of pkt_payload size. Each
4457 * packet is tagged with an unique packet ID in
4458 * the case of a reliable link.
4460 * On return, size contains the number of bytes written.
4461 * This function needs to ensure that the write size is < MTU size
4464 i_ldc_write_packet(ldc_chan_t
*ldcp
, caddr_t buf
, size_t *size
)
4467 uint64_t tx_head
, tx_tail
, new_tail
, start
;
4468 uint64_t txq_size_mask
, numavail
;
4469 uint8_t *msgbuf
, *source
= (uint8_t *)buf
;
4470 size_t len
, bytes_written
= 0, remaining
;
4472 uint32_t curr_seqid
;
4474 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
4476 ASSERT(ldcp
->mode
== LDC_MODE_RELIABLE
||
4477 ldcp
->mode
== LDC_MODE_UNRELIABLE
);
4479 /* compute mask for increment */
4480 txq_size_mask
= (ldcp
->tx_q_entries
- 1) << LDC_PACKET_SHIFT
;
4482 /* get the qptrs for the tx queue */
4483 rv
= hv_ldc_tx_get_state(ldcp
->id
,
4484 &ldcp
->tx_head
, &ldcp
->tx_tail
, &ldcp
->link_state
);
4487 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp
->id
);
4492 if (ldcp
->link_state
== LDC_CHANNEL_DOWN
||
4493 ldcp
->link_state
== LDC_CHANNEL_RESET
) {
4495 "ldc_write: (0x%llx) channel down/reset\n", ldcp
->id
);
4497 if (mutex_tryenter(&ldcp
->lock
)) {
4498 i_ldc_reset(ldcp
, B_FALSE
);
4499 mutex_exit(&ldcp
->lock
);
4502 * Release Tx lock, and then reacquire channel
4503 * and Tx lock in correct order
4505 mutex_exit(&ldcp
->tx_lock
);
4506 mutex_enter(&ldcp
->lock
);
4507 mutex_enter(&ldcp
->tx_lock
);
4508 i_ldc_reset(ldcp
, B_FALSE
);
4509 mutex_exit(&ldcp
->lock
);
4511 return (ECONNRESET
);
4514 tx_tail
= ldcp
->tx_tail
;
4515 new_tail
= (tx_tail
+ LDC_PACKET_SIZE
) %
4516 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
);
4519 * Check to see if the queue is full. The check is done using
4520 * the appropriate head based on the link mode.
4522 i_ldc_get_tx_head(ldcp
, &tx_head
);
4524 if (new_tail
== tx_head
) {
4526 "ldc_write: (0x%llx) TX queue is full\n", ldcp
->id
);
4528 return (EWOULDBLOCK
);
4532 * Make sure that the LDC Tx queue has enough space
4534 numavail
= (tx_head
>> LDC_PACKET_SHIFT
) - (tx_tail
>> LDC_PACKET_SHIFT
)
4535 + ldcp
->tx_q_entries
- 1;
4536 numavail
%= ldcp
->tx_q_entries
;
4538 if (*size
> (numavail
* ldcp
->pkt_payload
)) {
4540 "ldc_write: (0x%llx) TX queue has no space\n", ldcp
->id
);
4541 return (EWOULDBLOCK
);
4544 D2(ldcp
->id
, "ldc_write: (0x%llx) start xfer size=%d",
4547 /* Send the data now */
4549 curr_seqid
= ldcp
->last_msg_snt
;
4552 while (*size
> bytes_written
) {
4554 ldcmsg
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ tx_tail
);
4556 msgbuf
= (uint8_t *)((ldcp
->mode
== LDC_MODE_RELIABLE
) ?
4557 ldcmsg
->rdata
: ldcmsg
->udata
);
4559 ldcmsg
->type
= LDC_DATA
;
4560 ldcmsg
->stype
= LDC_INFO
;
4563 remaining
= *size
- bytes_written
;
4564 len
= min(ldcp
->pkt_payload
, remaining
);
4565 ldcmsg
->env
= (uint8_t)len
;
4568 ldcmsg
->seqid
= curr_seqid
;
4570 /* copy the data into pkt */
4571 bcopy(source
, msgbuf
, len
);
4574 bytes_written
+= len
;
4576 /* increment tail */
4577 tx_tail
= (tx_tail
+ LDC_PACKET_SIZE
) & txq_size_mask
;
4579 ASSERT(tx_tail
!= tx_head
);
4582 /* Set the start and stop bits */
4583 ldcmsg
->env
|= LDC_FRAG_STOP
;
4584 ldcmsg
= (ldc_msg_t
*)(ldcp
->tx_q_va
+ start
);
4585 ldcmsg
->env
|= LDC_FRAG_START
;
4588 * All packets have been copied into the TX queue
4589 * update the tail ptr in the HV
4591 rv
= i_ldc_set_tx_tail(ldcp
, tx_tail
);
4593 ldcp
->tx_tail
= tx_tail
;
4594 ldcp
->last_msg_snt
= curr_seqid
;
4595 *size
= bytes_written
;
4599 if (rv
!= EWOULDBLOCK
) {
4601 if (mutex_tryenter(&ldcp
->lock
)) {
4602 i_ldc_reset(ldcp
, B_FALSE
);
4603 mutex_exit(&ldcp
->lock
);
4606 * Release Tx lock, and then reacquire channel
4607 * and Tx lock in correct order
4609 mutex_exit(&ldcp
->tx_lock
);
4610 mutex_enter(&ldcp
->lock
);
4611 mutex_enter(&ldcp
->tx_lock
);
4612 i_ldc_reset(ldcp
, B_FALSE
);
4613 mutex_exit(&ldcp
->lock
);
4615 return (ECONNRESET
);
4618 D1(ldcp
->id
, "hv_tx_set_tail returns 0x%x (head 0x%x, "
4619 "old tail 0x%x, new tail 0x%x, qsize=0x%x)\n",
4620 rv
, ldcp
->tx_head
, ldcp
->tx_tail
, tx_tail
,
4621 (ldcp
->tx_q_entries
<< LDC_PACKET_SHIFT
));
4623 rv2
= hv_ldc_tx_get_state(ldcp
->id
,
4624 &tx_head
, &tx_tail
, &ldcp
->link_state
);
4626 D1(ldcp
->id
, "hv_ldc_tx_get_state returns 0x%x "
4627 "(head 0x%x, tail 0x%x state 0x%x)\n",
4628 rv2
, tx_head
, tx_tail
, ldcp
->link_state
);
4633 D2(ldcp
->id
, "ldc_write: (0x%llx) end xfer size=%d", ldcp
->id
, *size
);
4639 * Write specified amount of bytes to the channel
4640 * in multiple pkts of pkt_payload size. Each
4641 * packet is tagged with an unique packet ID in
4642 * the case of a reliable link.
4644 * On return, size contains the number of bytes written.
4645 * This function needs to ensure that the write size is < MTU size
4648 i_ldc_write_stream(ldc_chan_t
*ldcp
, caddr_t buf
, size_t *sizep
)
4650 ASSERT(MUTEX_HELD(&ldcp
->tx_lock
));
4651 ASSERT(ldcp
->mode
== LDC_MODE_RELIABLE
);
4653 /* Truncate packet to max of MTU size */
4654 if (*sizep
> ldcp
->mtu
) *sizep
= ldcp
->mtu
;
4655 return (i_ldc_write_packet(ldcp
, buf
, sizep
));
4660 * Interfaces for channel nexus to register/unregister with LDC module
4661 * The nexus will register functions to be used to register individual
4662 * channels with the nexus and enable interrupts for the channels
4665 ldc_register(ldc_cnex_t
*cinfo
)
4669 if (cinfo
== NULL
|| cinfo
->dip
== NULL
||
4670 cinfo
->reg_chan
== NULL
|| cinfo
->unreg_chan
== NULL
||
4671 cinfo
->add_intr
== NULL
|| cinfo
->rem_intr
== NULL
||
4672 cinfo
->clr_intr
== NULL
) {
4674 DWARN(DBG_ALL_LDCS
, "ldc_register: invalid nexus info\n");
4678 mutex_enter(&ldcssp
->lock
);
4680 /* nexus registration */
4681 ldcssp
->cinfo
.dip
= cinfo
->dip
;
4682 ldcssp
->cinfo
.reg_chan
= cinfo
->reg_chan
;
4683 ldcssp
->cinfo
.unreg_chan
= cinfo
->unreg_chan
;
4684 ldcssp
->cinfo
.add_intr
= cinfo
->add_intr
;
4685 ldcssp
->cinfo
.rem_intr
= cinfo
->rem_intr
;
4686 ldcssp
->cinfo
.clr_intr
= cinfo
->clr_intr
;
4688 /* register any channels that might have been previously initialized */
4689 ldcp
= ldcssp
->chan_list
;
4691 if ((ldcp
->tstate
& TS_QCONF_RDY
) &&
4692 (ldcp
->tstate
& TS_CNEX_RDY
) == 0)
4693 (void) i_ldc_register_channel(ldcp
);
4698 mutex_exit(&ldcssp
->lock
);
4704 ldc_unregister(ldc_cnex_t
*cinfo
)
4706 if (cinfo
== NULL
|| cinfo
->dip
== NULL
) {
4707 DWARN(DBG_ALL_LDCS
, "ldc_unregister: invalid nexus info\n");
4711 mutex_enter(&ldcssp
->lock
);
4713 if (cinfo
->dip
!= ldcssp
->cinfo
.dip
) {
4714 DWARN(DBG_ALL_LDCS
, "ldc_unregister: invalid dip\n");
4715 mutex_exit(&ldcssp
->lock
);
4719 /* nexus unregister */
4720 ldcssp
->cinfo
.dip
= NULL
;
4721 ldcssp
->cinfo
.reg_chan
= NULL
;
4722 ldcssp
->cinfo
.unreg_chan
= NULL
;
4723 ldcssp
->cinfo
.add_intr
= NULL
;
4724 ldcssp
->cinfo
.rem_intr
= NULL
;
4725 ldcssp
->cinfo
.clr_intr
= NULL
;
4727 mutex_exit(&ldcssp
->lock
);
4733 ldc_info(ldc_handle_t handle
, ldc_info_t
*info
)
4738 if (handle
== NULL
|| info
== NULL
) {
4739 DWARN(DBG_ALL_LDCS
, "ldc_get_info: invalid args\n");
4743 ldcp
= (ldc_chan_t
*)handle
;
4745 mutex_enter(&ldcp
->lock
);
4747 /* check to see if channel is initalized */
4748 if ((ldcp
->tstate
& ~TS_IN_RESET
) < TS_INIT
) {
4750 "ldc_get_info: (0x%llx) channel not initialized\n",
4752 mutex_exit(&ldcp
->lock
);
4756 mutex_exit(&ldcp
->lock
);
4759 * ldcssp->mapin_size is the max amount of shared memory supported by
4760 * the Hypervisor per guest. e.g, legacy HV supports 64MB; latest HV
4761 * support 1GB. This size is read during ldc module initialization.
4763 * ldc_dring_direct_map_rsvd is the amount of memory reserved for
4764 * mapping in descriptor rings. In the initial implementation, we use a
4765 * simple approach to determine the amount of mapin space available per
4766 * channel. In future, we may implement strict accounting of the actual
4767 * memory consumed to determine the exact amount available per channel.
4769 if (ldcssp
->mapin_size
<= ldc_dring_direct_map_rsvd
) {
4770 info
->direct_map_size_max
= 0;
4774 avail
= ldcssp
->mapin_size
- ldc_dring_direct_map_rsvd
;
4775 if (avail
>= ldc_direct_map_size_max
) {
4776 info
->direct_map_size_max
= ldc_direct_map_size_max
;
4778 info
->direct_map_size_max
= 0;