4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
28 * Logical domain channel devices are devices implemented entirely
29 * in software; cnex is the nexus for channel-devices. They use
30 * the HV channel interfaces via the LDC transport module to send
31 * and receive data and to register callbacks.
34 #include <sys/types.h>
35 #include <sys/cmn_err.h>
38 #include <sys/ddi_impldefs.h>
39 #include <sys/devops.h>
40 #include <sys/instance.h>
41 #include <sys/modctl.h>
44 #include <sys/sunddi.h>
45 #include <sys/sunndi.h>
46 #include <sys/systm.h>
47 #include <sys/mkdev.h>
48 #include <sys/machsystm.h>
49 #include <sys/intreg.h>
51 #include <sys/ddi_intr_impl.h>
52 #include <sys/ivintr.h>
53 #include <sys/hypervisor_api.h>
56 #include <sys/mach_descrip.h>
61 * Internal functions/information
63 static struct cnex_intr_map cnex_class_to_intr
[] = {
64 {LDC_DEV_GENERIC
, PIL_3
, 0},
65 {LDC_DEV_BLK
, PIL_4
, 10},
66 {LDC_DEV_BLK_SVC
, PIL_3
, 10},
67 {LDC_DEV_NT
, PIL_6
, 35},
68 {LDC_DEV_NT_SVC
, PIL_4
, 35},
69 {LDC_DEV_SERIAL
, PIL_6
, 0}
71 #define CNEX_MAX_DEVS (sizeof (cnex_class_to_intr) / \
72 sizeof (cnex_class_to_intr[0]))
74 #define CNEX_TX_INTR_WEIGHT 0
76 #define SUN4V_REG_SPEC2CFG_HDL(x) ((x >> 32) & ~(0xfull << 28))
78 static clock_t cnex_wait_usecs
= 1000; /* wait time in usecs */
79 static int cnex_wait_retries
= 3;
80 static void *cnex_state
;
82 static uint_t
cnex_intr_wrapper(caddr_t arg
);
83 static dev_info_t
*cnex_find_chan_dip(dev_info_t
*dip
, uint64_t chan_id
,
84 md_t
*mdp
, mde_cookie_t mde
);
87 * Channel Interrupt Distribution
89 * In order to balance interrupts among available CPUs, we use
90 * the intr_dist_cpuid_{add,remove}_device_weight() interface to
91 * assign weights to channel interrupts. These weights, which are
92 * defined in the cnex_intr_map structure, influence which CPU
93 * is returned by intr_dist_cpuid() when called via the cnex
94 * interrupt redistribution callback cnex_intr_redist().
95 * Interrupts for VIO devclass channels are given more weight than
96 * other interrupts because they are expected to occur more
97 * frequently and have a larger impact on overall performance.
98 * Transmit interrupts are given a zero weight because they are
101 * The interrupt weights influence the target CPU selection when
102 * interrupts are redistributed and when they are added. However,
103 * removal of interrupts can unbalance the distribution even if
104 * they are removed in converse order--compared to the order they
105 * are added. This can occur when interrupts are removed after
106 * redistribution occurs.
108 * Channel interrupt weights affect interrupt-CPU distribution
109 * relative to other weighted interrupts on the system. For VIO
110 * devclass channels, values are chosen to match those used by
111 * the PCI express nexus driver for net and storage devices.
113 static void cnex_intr_redist(void *arg
, int32_t weight_max
, int32_t weight
);
114 static int cnex_intr_new_cpu(cnex_soft_state_t
*ssp
, cnex_intr_t
*iinfo
);
115 static int cnex_intr_dis_wait(cnex_soft_state_t
*ssp
, cnex_intr_t
*iinfo
);
116 static int32_t cnex_class_weight(ldc_dev_t devclass
);
124 * Print debug messages
126 * set cnexdbg to 0xf for enabling all msgs
129 * 0x2 - All debug messages
130 * 0x1 - Minimal debug messages
136 cnexdebug(const char *fmt
, ...)
142 (void) vsprintf(buf
, fmt
, ap
);
145 cmn_err(CE_CONT
, "%s\n", buf
);
149 if (cnexdbg & 0x01) \
153 if (cnexdbg & 0x02) \
157 if (cnexdbg & 0x04) \
161 if (cnexdbg & 0x08) \
176 static int cnex_attach(dev_info_t
*, ddi_attach_cmd_t
);
177 static int cnex_detach(dev_info_t
*, ddi_detach_cmd_t
);
178 static int cnex_open(dev_t
*, int, int, cred_t
*);
179 static int cnex_close(dev_t
, int, int, cred_t
*);
180 static int cnex_ioctl(dev_t
, int, intptr_t, int, cred_t
*, int *);
181 static int cnex_ctl(dev_info_t
*, dev_info_t
*, ddi_ctl_enum_t
, void *,
184 static struct bus_ops cnex_bus_ops
= {
186 nullbusmap
, /* bus_map */
187 NULL
, /* bus_get_intrspec */
188 NULL
, /* bus_add_intrspec */
189 NULL
, /* bus_remove_intrspec */
190 i_ddi_map_fault
, /* bus_map_fault */
191 ddi_no_dma_map
, /* bus_dma_map */
192 ddi_no_dma_allochdl
, /* bus_dma_allochdl */
193 NULL
, /* bus_dma_freehdl */
194 NULL
, /* bus_dma_bindhdl */
195 NULL
, /* bus_dma_unbindhdl */
196 NULL
, /* bus_dma_flush */
197 NULL
, /* bus_dma_win */
198 NULL
, /* bus_dma_ctl */
199 cnex_ctl
, /* bus_ctl */
200 ddi_bus_prop_op
, /* bus_prop_op */
201 0, /* bus_get_eventcookie */
202 0, /* bus_add_eventcall */
203 0, /* bus_remove_eventcall */
204 0, /* bus_post_event */
205 NULL
, /* bus_intr_ctl */
206 NULL
, /* bus_config */
207 NULL
, /* bus_unconfig */
208 NULL
, /* bus_fm_init */
209 NULL
, /* bus_fm_fini */
210 NULL
, /* bus_fm_access_enter */
211 NULL
, /* bus_fm_access_exit */
212 NULL
, /* bus_power */
213 NULL
/* bus_intr_op */
216 static struct cb_ops cnex_cb_ops
= {
217 cnex_open
, /* open */
218 cnex_close
, /* close */
219 nodev
, /* strategy */
224 cnex_ioctl
, /* ioctl */
229 ddi_prop_op
, /* cb_prop_op */
231 D_MP
| D_NEW
| D_HOTPLUG
/* Driver compatibility flag */
234 static struct dev_ops cnex_ops
= {
235 DEVO_REV
, /* devo_rev, */
237 ddi_getinfo_1to1
, /* info */
238 nulldev
, /* identify */
240 cnex_attach
, /* attach */
241 cnex_detach
, /* detach */
243 &cnex_cb_ops
, /* driver operations */
244 &cnex_bus_ops
, /* bus operations */
246 ddi_quiesce_not_needed
, /* quiesce */
250 * Module linkage information for the kernel.
252 static struct modldrv modldrv
= {
254 "sun4v channel-devices nexus",
258 static struct modlinkage modlinkage
= {
259 MODREV_1
, (void *)&modldrv
, NULL
270 * Check HV intr group api versioning.
271 * Note that cnex assumes interrupt cookies is
272 * in version 1.0 of the intr group api.
274 if ((err
= hsvc_version(HSVC_GROUP_INTR
, &majornum
, &minornum
)) != 0) {
275 cmn_err(CE_WARN
, "cnex: failed to get intr api "
276 "group versioning errno=%d", err
);
278 } else if ((majornum
!= 1) && (majornum
!= 2)) {
279 cmn_err(CE_WARN
, "cnex: unsupported intr api group: "
280 "maj:0x%lx, min:0x%lx", majornum
, minornum
);
284 if ((err
= ddi_soft_state_init(&cnex_state
,
285 sizeof (cnex_soft_state_t
), 0)) != 0) {
288 if ((err
= mod_install(&modlinkage
)) != 0) {
289 ddi_soft_state_fini(&cnex_state
);
300 if ((err
= mod_remove(&modlinkage
)) != 0)
302 ddi_soft_state_fini(&cnex_state
);
307 _info(struct modinfo
*modinfop
)
309 return (mod_info(&modlinkage
, modinfop
));
313 * Callback function invoked by the interrupt redistribution
314 * framework. This will redirect interrupts at CPUs that are
315 * currently available in the system.
317 * Note: any interrupts with weight greater than or equal to
318 * weight_max must be redistributed when this callback is
319 * invoked with (weight == weight_max) which will be once per
324 cnex_intr_redist(void *arg
, int32_t weight_max
, int32_t weight
)
327 cnex_soft_state_t
*cnex_ssp
= arg
;
329 ASSERT(cnex_ssp
!= NULL
);
330 mutex_enter(&cnex_ssp
->clist_lock
);
332 cldcp
= cnex_ssp
->clist
;
333 while (cldcp
!= NULL
) {
335 mutex_enter(&cldcp
->lock
);
337 if (cldcp
->tx
.hdlr
&& (cldcp
->tx
.weight
== weight
||
338 (weight_max
== weight
&& cldcp
->tx
.weight
> weight
))) {
339 (void) cnex_intr_new_cpu(cnex_ssp
, &cldcp
->tx
);
342 if (cldcp
->rx
.hdlr
&& (cldcp
->rx
.weight
== weight
||
343 (weight_max
== weight
&& cldcp
->rx
.weight
> weight
))) {
344 (void) cnex_intr_new_cpu(cnex_ssp
, &cldcp
->rx
);
347 mutex_exit(&cldcp
->lock
);
353 mutex_exit(&cnex_ssp
->clist_lock
);
357 * Internal function to replace the CPU used by an interrupt
358 * during interrupt redistribution.
361 cnex_intr_new_cpu(cnex_soft_state_t
*ssp
, cnex_intr_t
*iinfo
)
366 /* Determine if the interrupt is enabled */
367 rv
= hvldc_intr_getvalid(ssp
->cfghdl
, iinfo
->ino
, &intr_state
);
369 DWARN("cnex_intr_new_cpu: rx ino=0x%llx, can't get valid\n",
374 /* If it is enabled, disable it */
375 if (intr_state
== HV_INTR_VALID
) {
376 rv
= cnex_intr_dis_wait(ssp
, iinfo
);
382 /* Target the interrupt at a new CPU. */
383 iinfo
->cpuid
= intr_dist_cpuid();
384 (void) hvldc_intr_settarget(ssp
->cfghdl
, iinfo
->ino
, iinfo
->cpuid
);
385 intr_dist_cpuid_add_device_weight(iinfo
->cpuid
, iinfo
->dip
,
388 /* Re-enable the interrupt if it was enabled */
389 if (intr_state
== HV_INTR_VALID
) {
390 (void) hvldc_intr_setvalid(ssp
->cfghdl
, iinfo
->ino
,
398 * Internal function to disable an interrupt and wait
399 * for any pending interrupts to finish.
402 cnex_intr_dis_wait(cnex_soft_state_t
*ssp
, cnex_intr_t
*iinfo
)
404 int rv
, intr_state
, retries
;
406 /* disable interrupts */
407 rv
= hvldc_intr_setvalid(ssp
->cfghdl
, iinfo
->ino
, HV_INTR_NOTVALID
);
409 DWARN("cnex_intr_dis_wait: ino=0x%llx, can't set valid\n",
415 * Make a best effort to wait for pending interrupts
416 * to finish. There is not much we can do if we timeout.
421 rv
= hvldc_intr_getstate(ssp
->cfghdl
, iinfo
->ino
, &intr_state
);
423 DWARN("cnex_intr_dis_wait: ino=0x%llx, can't get "
424 "state\n", iinfo
->ino
);
428 if (intr_state
!= HV_INTR_DELIVERED_STATE
)
431 drv_usecwait(cnex_wait_usecs
);
433 } while (!panicstr
&& ++retries
<= cnex_wait_retries
);
439 * Returns the interrupt weight to use for the specified devclass.
442 cnex_class_weight(ldc_dev_t devclass
)
446 for (idx
= 0; idx
< CNEX_MAX_DEVS
; idx
++) {
447 if (devclass
== cnex_class_to_intr
[idx
].devclass
) {
448 return (cnex_class_to_intr
[idx
].weight
);
453 * If this code is reached, the specified devclass is
454 * invalid. New devclasses should be added to
455 * cnex_class_to_intr.
463 * Exported interface to register a LDC endpoint with
467 cnex_reg_chan(dev_info_t
*dip
, uint64_t id
, ldc_dev_t devclass
)
471 cnex_ldc_t
*new_cldcp
;
472 int listsz
, num_nodes
, num_channels
;
474 mde_cookie_t rootnode
, *listp
= NULL
;
476 uint64_t rxino
= (uint64_t)-1;
477 uint64_t txino
= (uint64_t)-1;
478 cnex_soft_state_t
*cnex_ssp
;
479 int status
, instance
;
480 dev_info_t
*chan_dip
= NULL
;
482 /* Get device instance and structure */
483 instance
= ddi_get_instance(dip
);
484 cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
);
486 /* Check to see if channel is already registered */
487 mutex_enter(&cnex_ssp
->clist_lock
);
488 cldcp
= cnex_ssp
->clist
;
490 if (cldcp
->id
== id
) {
491 DWARN("cnex_reg_chan: channel 0x%llx exists\n", id
);
492 mutex_exit(&cnex_ssp
->clist_lock
);
497 mutex_exit(&cnex_ssp
->clist_lock
);
499 /* Get the Tx/Rx inos from the MD */
500 if ((mdp
= md_get_handle()) == NULL
) {
501 DWARN("cnex_reg_chan: cannot init MD\n");
504 num_nodes
= md_node_count(mdp
);
505 ASSERT(num_nodes
> 0);
507 listsz
= num_nodes
* sizeof (mde_cookie_t
);
508 listp
= (mde_cookie_t
*)kmem_zalloc(listsz
, KM_SLEEP
);
510 rootnode
= md_root_node(mdp
);
512 /* search for all channel_endpoint nodes */
513 num_channels
= md_scan_dag(mdp
, rootnode
,
514 md_find_name(mdp
, "channel-endpoint"),
515 md_find_name(mdp
, "fwd"), listp
);
516 if (num_channels
<= 0) {
517 DWARN("cnex_reg_chan: invalid channel id\n");
518 kmem_free(listp
, listsz
);
519 (void) md_fini_handle(mdp
);
523 for (idx
= 0; idx
< num_channels
; idx
++) {
525 /* Get the channel ID */
526 status
= md_get_prop_val(mdp
, listp
[idx
], "id", &tmp_id
);
528 DWARN("cnex_reg_chan: cannot read LDC ID\n");
529 kmem_free(listp
, listsz
);
530 (void) md_fini_handle(mdp
);
536 /* Get the Tx and Rx ino */
537 status
= md_get_prop_val(mdp
, listp
[idx
], "tx-ino", &txino
);
539 DWARN("cnex_reg_chan: cannot read Tx ino\n");
540 kmem_free(listp
, listsz
);
541 (void) md_fini_handle(mdp
);
544 status
= md_get_prop_val(mdp
, listp
[idx
], "rx-ino", &rxino
);
546 DWARN("cnex_reg_chan: cannot read Rx ino\n");
547 kmem_free(listp
, listsz
);
548 (void) md_fini_handle(mdp
);
551 chan_dip
= cnex_find_chan_dip(dip
, id
, mdp
, listp
[idx
]);
552 ASSERT(chan_dip
!= NULL
);
554 kmem_free(listp
, listsz
);
555 (void) md_fini_handle(mdp
);
558 * check to see if we looped through the list of channel IDs without
559 * matching one (i.e. an 'ino' has not been initialised).
561 if ((rxino
== -1) || (txino
== -1)) {
562 DERR("cnex_reg_chan: no ID matching '%llx' in MD\n", id
);
566 /* Allocate a new channel structure */
567 new_cldcp
= kmem_zalloc(sizeof (*new_cldcp
), KM_SLEEP
);
569 /* Initialize the channel */
570 mutex_init(&new_cldcp
->lock
, NULL
, MUTEX_DRIVER
, NULL
);
573 new_cldcp
->tx
.ino
= txino
;
574 new_cldcp
->rx
.ino
= rxino
;
575 new_cldcp
->devclass
= devclass
;
576 new_cldcp
->tx
.weight
= CNEX_TX_INTR_WEIGHT
;
577 new_cldcp
->rx
.weight
= cnex_class_weight(devclass
);
578 new_cldcp
->dip
= chan_dip
;
581 * Add channel to nexus channel list.
582 * Check again to see if channel is already registered since
583 * clist_lock was dropped above.
585 mutex_enter(&cnex_ssp
->clist_lock
);
586 cldcp
= cnex_ssp
->clist
;
588 if (cldcp
->id
== id
) {
589 DWARN("cnex_reg_chan: channel 0x%llx exists\n", id
);
590 mutex_exit(&cnex_ssp
->clist_lock
);
591 mutex_destroy(&new_cldcp
->lock
);
592 kmem_free(new_cldcp
, sizeof (*new_cldcp
));
597 new_cldcp
->next
= cnex_ssp
->clist
;
598 cnex_ssp
->clist
= new_cldcp
;
599 mutex_exit(&cnex_ssp
->clist_lock
);
605 * Add Tx/Rx interrupt handler for the channel
608 cnex_add_intr(dev_info_t
*dip
, uint64_t id
, cnex_intrtype_t itype
,
609 uint_t (*hdlr
)(), caddr_t arg1
, caddr_t arg2
)
614 cnex_soft_state_t
*cnex_ssp
;
617 /* Get device instance and structure */
618 instance
= ddi_get_instance(dip
);
619 cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
);
621 /* get channel info */
622 mutex_enter(&cnex_ssp
->clist_lock
);
623 cldcp
= cnex_ssp
->clist
;
630 DWARN("cnex_add_intr: channel 0x%llx does not exist\n", id
);
631 mutex_exit(&cnex_ssp
->clist_lock
);
634 mutex_exit(&cnex_ssp
->clist_lock
);
636 /* get channel lock */
637 mutex_enter(&cldcp
->lock
);
639 /* get interrupt type */
640 if (itype
== CNEX_TX_INTR
) {
641 iinfo
= &(cldcp
->tx
);
642 } else if (itype
== CNEX_RX_INTR
) {
643 iinfo
= &(cldcp
->rx
);
645 DWARN("cnex_add_intr: invalid interrupt type\n", id
);
646 mutex_exit(&cldcp
->lock
);
650 /* check if a handler is already added */
651 if (iinfo
->hdlr
!= 0) {
652 DWARN("cnex_add_intr: interrupt handler exists\n");
653 mutex_exit(&cldcp
->lock
);
657 /* save interrupt handler info */
662 /* save data for DTrace probes used by intrstat(1m) */
663 iinfo
->dip
= cldcp
->dip
;
664 iinfo
->id
= cldcp
->id
;
666 iinfo
->icookie
= MINVINTR_COOKIE
+ iinfo
->ino
;
669 * Verify that the ino does not generate a cookie which
670 * is outside the (MINVINTR_COOKIE, MAXIVNUM) range of the
671 * system interrupt table.
673 if (iinfo
->icookie
>= MAXIVNUM
|| iinfo
->icookie
< MINVINTR_COOKIE
) {
674 DWARN("cnex_add_intr: invalid cookie %x ino %x\n",
675 iinfo
->icookie
, iinfo
->ino
);
676 mutex_exit(&cldcp
->lock
);
680 D1("cnex_add_intr: add hdlr, cfghdl=0x%llx, ino=0x%llx, "
681 "cookie=0x%llx\n", cnex_ssp
->cfghdl
, iinfo
->ino
, iinfo
->icookie
);
683 /* Pick a PIL on the basis of the channel's devclass */
684 for (idx
= 0, pil
= PIL_3
; idx
< CNEX_MAX_DEVS
; idx
++) {
685 if (cldcp
->devclass
== cnex_class_to_intr
[idx
].devclass
) {
686 pil
= cnex_class_to_intr
[idx
].pil
;
691 /* add interrupt to solaris ivec table */
692 if (add_ivintr(iinfo
->icookie
, pil
, (intrfunc
)cnex_intr_wrapper
,
693 (caddr_t
)iinfo
, NULL
, NULL
) != 0) {
694 DWARN("cnex_add_intr: add_ivintr fail cookie %x ino %x\n",
695 iinfo
->icookie
, iinfo
->ino
);
696 mutex_exit(&cldcp
->lock
);
700 /* set the cookie in the HV */
701 rv
= hvldc_intr_setcookie(cnex_ssp
->cfghdl
, iinfo
->ino
, iinfo
->icookie
);
703 /* pick next CPU in the domain for this channel */
704 iinfo
->cpuid
= intr_dist_cpuid();
706 /* set the target CPU and then enable interrupts */
707 rv
= hvldc_intr_settarget(cnex_ssp
->cfghdl
, iinfo
->ino
, iinfo
->cpuid
);
709 DWARN("cnex_add_intr: ino=0x%llx, cannot set target cpu\n",
713 rv
= hvldc_intr_setstate(cnex_ssp
->cfghdl
, iinfo
->ino
,
716 DWARN("cnex_add_intr: ino=0x%llx, cannot set state\n",
720 rv
= hvldc_intr_setvalid(cnex_ssp
->cfghdl
, iinfo
->ino
, HV_INTR_VALID
);
722 DWARN("cnex_add_intr: ino=0x%llx, cannot set valid\n",
727 intr_dist_cpuid_add_device_weight(iinfo
->cpuid
, iinfo
->dip
,
730 mutex_exit(&cldcp
->lock
);
734 (void) rem_ivintr(iinfo
->icookie
, pil
);
735 mutex_exit(&cldcp
->lock
);
741 * Exported interface to unregister a LDC endpoint with
745 cnex_unreg_chan(dev_info_t
*dip
, uint64_t id
)
747 cnex_ldc_t
*cldcp
, *prev_cldcp
;
748 cnex_soft_state_t
*cnex_ssp
;
751 /* Get device instance and structure */
752 instance
= ddi_get_instance(dip
);
753 cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
);
755 /* find and remove channel from list */
756 mutex_enter(&cnex_ssp
->clist_lock
);
758 cldcp
= cnex_ssp
->clist
;
767 DWARN("cnex_unreg_chan: invalid channel %d\n", id
);
768 mutex_exit(&cnex_ssp
->clist_lock
);
772 if (cldcp
->tx
.hdlr
|| cldcp
->rx
.hdlr
) {
773 DWARN("cnex_unreg_chan: handlers still exist: chan %lx\n", id
);
774 mutex_exit(&cnex_ssp
->clist_lock
);
779 prev_cldcp
->next
= cldcp
->next
;
781 cnex_ssp
->clist
= cldcp
->next
;
783 mutex_exit(&cnex_ssp
->clist_lock
);
786 mutex_destroy(&cldcp
->lock
);
789 kmem_free(cldcp
, sizeof (*cldcp
));
795 * Remove Tx/Rx interrupt handler for the channel
798 cnex_rem_intr(dev_info_t
*dip
, uint64_t id
, cnex_intrtype_t itype
)
803 cnex_soft_state_t
*cnex_ssp
;
804 int instance
, istate
;
806 /* Get device instance and structure */
807 instance
= ddi_get_instance(dip
);
808 cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
);
810 /* get channel info */
811 mutex_enter(&cnex_ssp
->clist_lock
);
812 cldcp
= cnex_ssp
->clist
;
819 DWARN("cnex_rem_intr: channel 0x%llx does not exist\n", id
);
820 mutex_exit(&cnex_ssp
->clist_lock
);
823 mutex_exit(&cnex_ssp
->clist_lock
);
825 /* get rid of the channel intr handler */
826 mutex_enter(&cldcp
->lock
);
828 /* get interrupt type */
829 if (itype
== CNEX_TX_INTR
) {
830 iinfo
= &(cldcp
->tx
);
831 } else if (itype
== CNEX_RX_INTR
) {
832 iinfo
= &(cldcp
->rx
);
834 DWARN("cnex_rem_intr: invalid interrupt type\n");
835 mutex_exit(&cldcp
->lock
);
839 D1("cnex_rem_intr: interrupt ino=0x%x\n", iinfo
->ino
);
841 /* check if a handler is already added */
842 if (iinfo
->hdlr
== 0) {
843 DWARN("cnex_rem_intr: interrupt handler does not exist\n");
844 mutex_exit(&cldcp
->lock
);
848 D1("cnex_rem_intr: set intr to invalid ino=0x%x\n", iinfo
->ino
);
849 rv
= hvldc_intr_setvalid(cnex_ssp
->cfghdl
,
850 iinfo
->ino
, HV_INTR_NOTVALID
);
852 DWARN("cnex_rem_intr: cannot set valid ino=%x\n", iinfo
->ino
);
853 mutex_exit(&cldcp
->lock
);
858 * Check if there are pending interrupts. If interrupts are
859 * pending return EAGAIN.
861 rv
= hvldc_intr_getstate(cnex_ssp
->cfghdl
, iinfo
->ino
, &istate
);
863 DWARN("cnex_rem_intr: ino=0x%llx, cannot get state\n",
865 mutex_exit(&cldcp
->lock
);
869 /* if interrupts are still pending print warning */
870 if (istate
!= HV_INTR_IDLE_STATE
) {
871 DWARN("cnex_rem_intr: cannot remove intr busy ino=%x\n",
873 mutex_exit(&cldcp
->lock
);
877 /* Pick a PIL on the basis of the channel's devclass */
878 for (idx
= 0, pil
= PIL_3
; idx
< CNEX_MAX_DEVS
; idx
++) {
879 if (cldcp
->devclass
== cnex_class_to_intr
[idx
].devclass
) {
880 pil
= cnex_class_to_intr
[idx
].pil
;
885 intr_dist_cpuid_rem_device_weight(iinfo
->cpuid
, iinfo
->dip
);
887 /* remove interrupt */
888 (void) rem_ivintr(iinfo
->icookie
, pil
);
890 /* clear interrupt info */
891 bzero(iinfo
, sizeof (*iinfo
));
893 mutex_exit(&cldcp
->lock
);
900 * Clear pending Tx/Rx interrupt
903 cnex_clr_intr(dev_info_t
*dip
, uint64_t id
, cnex_intrtype_t itype
)
908 cnex_soft_state_t
*cnex_ssp
;
911 /* Get device instance and structure */
912 instance
= ddi_get_instance(dip
);
913 cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
);
915 /* get channel info */
916 mutex_enter(&cnex_ssp
->clist_lock
);
917 cldcp
= cnex_ssp
->clist
;
924 DWARN("cnex_clr_intr: channel 0x%llx does not exist\n", id
);
925 mutex_exit(&cnex_ssp
->clist_lock
);
928 mutex_exit(&cnex_ssp
->clist_lock
);
930 mutex_enter(&cldcp
->lock
);
932 /* get interrupt type */
933 if (itype
== CNEX_TX_INTR
) {
934 iinfo
= &(cldcp
->tx
);
935 } else if (itype
== CNEX_RX_INTR
) {
936 iinfo
= &(cldcp
->rx
);
938 DWARN("cnex_clr_intr: invalid interrupt type\n");
939 mutex_exit(&cldcp
->lock
);
943 D1("%s: interrupt ino=0x%x\n", __func__
, iinfo
->ino
);
945 /* check if a handler is already added */
946 if (iinfo
->hdlr
== 0) {
947 DWARN("cnex_clr_intr: interrupt handler does not exist\n");
948 mutex_exit(&cldcp
->lock
);
952 rv
= hvldc_intr_setstate(cnex_ssp
->cfghdl
, iinfo
->ino
,
955 DWARN("cnex_clr_intr: cannot clear interrupt state\n");
956 mutex_exit(&cldcp
->lock
);
960 mutex_exit(&cldcp
->lock
);
966 * Channel nexus interrupt handler wrapper
969 cnex_intr_wrapper(caddr_t arg
)
973 caddr_t handler_arg1
;
974 caddr_t handler_arg2
;
975 cnex_intr_t
*iinfo
= (cnex_intr_t
*)arg
;
977 ASSERT(iinfo
!= NULL
);
979 handler
= iinfo
->hdlr
;
980 handler_arg1
= iinfo
->arg1
;
981 handler_arg2
= iinfo
->arg2
;
984 * The 'interrupt__start' and 'interrupt__complete' probes
985 * are provided to support 'intrstat' command. These probes
986 * help monitor the interrupts on a per device basis only.
987 * In order to provide the ability to monitor the
988 * activity on a per channel basis, two additional
989 * probes('channelintr__start','channelintr__complete')
992 DTRACE_PROBE4(channelintr__start
, uint64_t, iinfo
->id
,
993 cnex_intr_t
*, iinfo
, void *, handler
, caddr_t
, handler_arg1
);
995 DTRACE_PROBE4(interrupt__start
, dev_info_t
, iinfo
->dip
,
996 void *, handler
, caddr_t
, handler_arg1
, caddr_t
, handler_arg2
);
998 D1("cnex_intr_wrapper:ino=0x%llx invoke client handler\n", iinfo
->ino
);
999 res
= (*handler
)(handler_arg1
, handler_arg2
);
1001 DTRACE_PROBE4(interrupt__complete
, dev_info_t
, iinfo
->dip
,
1002 void *, handler
, caddr_t
, handler_arg1
, int, res
);
1004 DTRACE_PROBE4(channelintr__complete
, uint64_t, iinfo
->id
,
1005 cnex_intr_t
*, iinfo
, void *, handler
, caddr_t
, handler_arg1
);
1012 cnex_attach(dev_info_t
*devi
, ddi_attach_cmd_t cmd
)
1014 int rv
, instance
, reglen
;
1015 cnex_regspec_t
*reg_p
;
1017 cnex_soft_state_t
*cnex_ssp
;
1023 return (DDI_SUCCESS
);
1025 return (DDI_FAILURE
);
1029 * Get the instance specific soft state structure.
1030 * Save the devi for this instance in the soft_state data.
1032 instance
= ddi_get_instance(devi
);
1033 if (ddi_soft_state_zalloc(cnex_state
, instance
) != DDI_SUCCESS
)
1034 return (DDI_FAILURE
);
1035 cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
);
1037 cnex_ssp
->devi
= devi
;
1038 cnex_ssp
->clist
= NULL
;
1040 if (ddi_getlongprop(DDI_DEV_T_ANY
, devi
, DDI_PROP_DONTPASS
,
1041 "reg", (caddr_t
)®_p
, ®len
) != DDI_SUCCESS
) {
1042 return (DDI_FAILURE
);
1045 /* get the sun4v config handle for this device */
1046 cnex_ssp
->cfghdl
= SUN4V_REG_SPEC2CFG_HDL(reg_p
->physaddr
);
1047 kmem_free(reg_p
, reglen
);
1049 D1("cnex_attach: cfghdl=0x%llx\n", cnex_ssp
->cfghdl
);
1051 /* init channel list mutex */
1052 mutex_init(&cnex_ssp
->clist_lock
, NULL
, MUTEX_DRIVER
, NULL
);
1054 /* Register with LDC module */
1056 cinfo
.reg_chan
= cnex_reg_chan
;
1057 cinfo
.unreg_chan
= cnex_unreg_chan
;
1058 cinfo
.add_intr
= cnex_add_intr
;
1059 cinfo
.rem_intr
= cnex_rem_intr
;
1060 cinfo
.clr_intr
= cnex_clr_intr
;
1063 * LDC register will fail if an nexus instance had already
1064 * registered with the LDC framework
1066 rv
= ldc_register(&cinfo
);
1068 DWARN("cnex_attach: unable to register with LDC\n");
1069 ddi_soft_state_free(cnex_state
, instance
);
1070 mutex_destroy(&cnex_ssp
->clist_lock
);
1071 return (DDI_FAILURE
);
1074 if (ddi_create_minor_node(devi
, "devctl", S_IFCHR
, instance
,
1075 DDI_NT_NEXUS
, 0) != DDI_SUCCESS
) {
1076 ddi_remove_minor_node(devi
, NULL
);
1077 ddi_soft_state_free(cnex_state
, instance
);
1078 mutex_destroy(&cnex_ssp
->clist_lock
);
1079 return (DDI_FAILURE
);
1082 /* Add interrupt redistribution callback. */
1083 intr_dist_add_weighted(cnex_intr_redist
, cnex_ssp
);
1085 ddi_report_dev(devi
);
1086 return (DDI_SUCCESS
);
1091 cnex_detach(dev_info_t
*devi
, ddi_detach_cmd_t cmd
)
1095 cnex_soft_state_t
*cnex_ssp
;
1101 return (DDI_SUCCESS
);
1103 return (DDI_FAILURE
);
1106 instance
= ddi_get_instance(devi
);
1107 cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
);
1109 /* check if there are any channels still registered */
1110 if (cnex_ssp
->clist
) {
1111 cmn_err(CE_WARN
, "?cnex_dettach: channels registered %d\n",
1112 ddi_get_instance(devi
));
1113 return (DDI_FAILURE
);
1116 /* Unregister with LDC module */
1118 (void) ldc_unregister(&cinfo
);
1120 /* Remove interrupt redistribution callback. */
1121 intr_dist_rem_weighted(cnex_intr_redist
, cnex_ssp
);
1124 mutex_destroy(&cnex_ssp
->clist_lock
);
1126 /* free soft state structure */
1127 ddi_soft_state_free(cnex_state
, instance
);
1129 return (DDI_SUCCESS
);
1134 cnex_open(dev_t
*devp
, int flags
, int otyp
, cred_t
*credp
)
1138 if (otyp
!= OTYP_CHR
)
1141 instance
= getminor(*devp
);
1142 if (ddi_get_soft_state(cnex_state
, instance
) == NULL
)
1150 cnex_close(dev_t dev
, int flags
, int otyp
, cred_t
*credp
)
1154 if (otyp
!= OTYP_CHR
)
1157 instance
= getminor(dev
);
1158 if (ddi_get_soft_state(cnex_state
, instance
) == NULL
)
1166 cnex_ioctl(dev_t dev
,
1167 int cmd
, intptr_t arg
, int mode
, cred_t
*cred_p
, int *rval_p
)
1170 cnex_soft_state_t
*cnex_ssp
;
1172 instance
= getminor(dev
);
1173 if ((cnex_ssp
= ddi_get_soft_state(cnex_state
, instance
)) == NULL
)
1175 ASSERT(cnex_ssp
->devi
);
1176 return (ndi_devctl_ioctl(cnex_ssp
->devi
, cmd
, arg
, mode
, 0));
1180 cnex_ctl(dev_info_t
*dip
, dev_info_t
*rdip
, ddi_ctl_enum_t ctlop
,
1181 void *arg
, void *result
)
1183 char name
[MAXNAMELEN
];
1188 case DDI_CTLOPS_REPORTDEV
:
1190 return (DDI_FAILURE
);
1191 cmn_err(CE_CONT
, "?channel-device: %s%d\n",
1192 ddi_driver_name(rdip
), ddi_get_instance(rdip
));
1193 return (DDI_SUCCESS
);
1195 case DDI_CTLOPS_INITCHILD
:
1197 dev_info_t
*child
= (dev_info_t
*)arg
;
1199 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY
, child
,
1200 DDI_PROP_DONTPASS
, "reg",
1201 &cnex_regspec
, ®len
) != DDI_SUCCESS
) {
1202 return (DDI_FAILURE
);
1205 (void) snprintf(name
, sizeof (name
), "%x", *cnex_regspec
);
1206 ddi_set_name_addr(child
, name
);
1207 ddi_set_parent_data(child
, NULL
);
1208 ddi_prop_free(cnex_regspec
);
1209 return (DDI_SUCCESS
);
1212 case DDI_CTLOPS_UNINITCHILD
:
1214 dev_info_t
*child
= (dev_info_t
*)arg
;
1216 NDI_CONFIG_DEBUG((CE_NOTE
,
1217 "DDI_CTLOPS_UNINITCHILD(%s, instance=%d)",
1218 ddi_driver_name(child
), DEVI(child
)->devi_instance
));
1220 ddi_set_name_addr(child
, NULL
);
1222 return (DDI_SUCCESS
);
1225 case DDI_CTLOPS_DMAPMAPC
:
1226 case DDI_CTLOPS_REPORTINT
:
1227 case DDI_CTLOPS_REGSIZE
:
1228 case DDI_CTLOPS_NREGS
:
1229 case DDI_CTLOPS_SIDDEV
:
1230 case DDI_CTLOPS_SLAVEONLY
:
1231 case DDI_CTLOPS_AFFINITY
:
1232 case DDI_CTLOPS_POKE
:
1233 case DDI_CTLOPS_PEEK
:
1235 * These ops correspond to functions that "shouldn't" be called
1236 * by a channel-device driver. So we whine when we're called.
1238 cmn_err(CE_WARN
, "%s%d: invalid op (%d) from %s%d\n",
1239 ddi_driver_name(dip
), ddi_get_instance(dip
), ctlop
,
1240 ddi_driver_name(rdip
), ddi_get_instance(rdip
));
1241 return (DDI_FAILURE
);
1243 case DDI_CTLOPS_ATTACH
:
1244 case DDI_CTLOPS_BTOP
:
1245 case DDI_CTLOPS_BTOPR
:
1246 case DDI_CTLOPS_DETACH
:
1247 case DDI_CTLOPS_DVMAPAGESIZE
:
1248 case DDI_CTLOPS_IOMIN
:
1249 case DDI_CTLOPS_POWER
:
1250 case DDI_CTLOPS_PTOB
:
1253 * Everything else (e.g. PTOB/BTOP/BTOPR requests) we pass up
1255 return (ddi_ctlops(dip
, rdip
, ctlop
, arg
, result
));
1260 * cnex_find_chan_dip -- Find the dip of a device that is corresponding
1261 * to the specific channel. Below are the details on how the dip
1264 * - In the MD, the cfg-handle is expected to be unique for
1265 * virtual-device nodes that have the same 'name' property value.
1266 * This value is expected to be the same as that of "reg" property
1267 * of the corresponding OBP device node.
1269 * - The value of the 'name' property of a virtual-device node
1270 * in the MD is expected to be the same for the corresponding
1273 * - Find the virtual-device node corresponding to a channel-endpoint
1274 * by walking backwards. Then obtain the values for the 'name' and
1275 * 'cfg-handle' properties.
1277 * - Walk all the children of the cnex, find a matching dip which
1278 * has the same 'name' and 'reg' property values.
1280 * - The channels that have no corresponding device driver are
1281 * treated as if they correspond to the cnex driver,
1282 * that is, return cnex dip for them. This means, the
1283 * cnex acts as an umbrella device driver. Note, this is
1284 * for 'intrstat' statistics purposes only. As a result of this,
1285 * the 'intrstat' shows cnex as the device that is servicing the
1286 * interrupts corresponding to these channels.
1288 * For now, only one such case is known, that is, the channels that
1289 * are used by the "domain-services".
1292 cnex_find_chan_dip(dev_info_t
*dip
, uint64_t chan_id
,
1293 md_t
*mdp
, mde_cookie_t mde
)
1300 mde_cookie_t
*listp
;
1301 dev_info_t
*cdip
= NULL
;
1303 num_nodes
= md_node_count(mdp
);
1304 ASSERT(num_nodes
> 0);
1305 listsz
= num_nodes
* sizeof (mde_cookie_t
);
1306 listp
= (mde_cookie_t
*)kmem_zalloc(listsz
, KM_SLEEP
);
1308 num_devs
= md_scan_dag(mdp
, mde
, md_find_name(mdp
, "virtual-device"),
1309 md_find_name(mdp
, "back"), listp
);
1310 ASSERT(num_devs
<= 1);
1311 if (num_devs
<= 0) {
1312 DWARN("cnex_find_chan_dip:channel(0x%llx): "
1313 "No virtual-device found\n", chan_id
);
1316 if (md_get_prop_str(mdp
, listp
[0], "name", &md_name
) != 0) {
1317 DWARN("cnex_find_chan_dip:channel(0x%llx): "
1318 "name property not found\n", chan_id
);
1322 D1("cnex_find_chan_dip: channel(0x%llx): virtual-device "
1323 "name property value = %s\n", chan_id
, md_name
);
1325 if (md_get_prop_val(mdp
, listp
[0], "cfg-handle", &cfghdl
) != 0) {
1326 DWARN("cnex_find_chan_dip:channel(0x%llx): virtual-device's "
1327 "cfg-handle property not found\n", chan_id
);
1331 D1("cnex_find_chan_dip:channel(0x%llx): virtual-device cfg-handle "
1332 " property value = 0x%x\n", chan_id
, cfghdl
);
1334 for (cdip
= ddi_get_child(dip
); cdip
!= NULL
;
1335 cdip
= ddi_get_next_sibling(cdip
)) {
1341 if (ddi_prop_lookup_string(DDI_DEV_T_ANY
, cdip
,
1342 DDI_PROP_DONTPASS
, "name",
1343 &dev_name
) != DDI_PROP_SUCCESS
) {
1344 DWARN("cnex_find_chan_dip: name property not"
1345 " found for dip(0x%p)\n", cdip
);
1348 if (strcmp(md_name
, dev_name
) != 0) {
1349 ddi_prop_free(dev_name
);
1352 ddi_prop_free(dev_name
);
1353 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY
, cdip
,
1354 DDI_PROP_DONTPASS
, "reg",
1355 &cnex_regspec
, ®len
) != DDI_SUCCESS
) {
1356 DWARN("cnex_find_chan_dip: reg property not"
1357 " found for dip(0x%p)\n", cdip
);
1360 if (*cnex_regspec
== cfghdl
) {
1361 D1("cnex_find_chan_dip:channel(0x%llx): found "
1362 "dip(0x%p) drvname=%s\n", chan_id
, cdip
,
1363 ddi_driver_name(cdip
));
1364 ddi_prop_free(cnex_regspec
);
1367 ddi_prop_free(cnex_regspec
);
1373 * If a virtual-device node exists but no dip found,
1374 * then for now print a DEBUG error message only.
1377 DERR("cnex_find_chan_dip:channel(0x%llx): "
1378 "No device found\n", chan_id
);
1381 /* If no dip was found, return cnex device's dip. */
1385 kmem_free(listp
, listsz
);
1386 D1("cnex_find_chan_dip:channel(0x%llx): returning dip=0x%p\n",
1391 /* -------------------------------------------------------------------------- */