4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2012 Garrett D'Amore <garrett@damore.org>
25 * Copyright 2014 Pluribus Networks, Inc.
26 * Copyright 2016 Nexenta Systems, Inc.
30 * PC specific DDI implementation
32 #include <sys/types.h>
33 #include <sys/autoconf.h>
34 #include <sys/avintr.h>
35 #include <sys/bootconf.h>
37 #include <sys/cpuvar.h>
38 #include <sys/ddi_impldefs.h>
39 #include <sys/ddi_subrdefs.h>
40 #include <sys/ethernet.h>
42 #include <sys/instance.h>
44 #include <sys/machsystm.h>
45 #include <sys/modctl.h>
46 #include <sys/promif.h>
47 #include <sys/prom_plat.h>
48 #include <sys/sunndi.h>
49 #include <sys/ndi_impldefs.h>
50 #include <sys/ddi_impldefs.h>
51 #include <sys/sysmacros.h>
52 #include <sys/systeminfo.h>
53 #include <sys/utsname.h>
54 #include <sys/atomic.h>
56 #include <sys/archsystm.h>
57 #include <vm/seg_kmem.h>
58 #include <sys/ontrap.h>
59 #include <sys/fm/protocol.h>
60 #include <sys/ramdisk.h>
61 #include <sys/sunndi.h>
63 #include <sys/pci_impl.h>
64 #include <sys/mach_intr.h>
65 #include <vm/hat_i86.h>
66 #include <sys/x86_archext.h>
70 * DDI Boot Configuration
74 * Platform drivers on this platform
76 char *platform_module_list
[] = {
82 /* pci bus resource maps */
83 struct pci_bus_resource
*pci_bus_res
;
85 size_t dma_max_copybuf_size
= 0x101000; /* 1M + 4K */
87 uint64_t ramdisk_start
, ramdisk_end
;
92 * Forward declarations
94 static int getlongprop_buf();
95 static void get_boot_properties(void);
96 static void impl_bus_initialprobe(void);
97 static void impl_bus_reprobe(void);
99 static int poke_mem(peekpoke_ctlops_t
*in_args
);
100 static int peek_mem(peekpoke_ctlops_t
*in_args
);
102 static int kmem_override_cache_attrs(caddr_t
, size_t, uint_t
);
105 extern void immu_init(void);
109 * We use an AVL tree to store contiguous address allocations made with the
110 * kalloca() routine, so that we can return the size to free with kfreea().
111 * Note that in the future it would be vastly faster if we could eliminate
112 * this lookup by insisting that all callers keep track of their own sizes,
113 * just as for kmem_alloc().
121 static avl_tree_t ctgtree
;
123 static kmutex_t ctgmutex
;
124 #define CTGLOCK() mutex_enter(&ctgmutex)
125 #define CTGUNLOCK() mutex_exit(&ctgmutex)
128 * Minimum pfn value of page_t's put on the free list. This is to simplify
129 * support of ddi dma memory requests which specify small, non-zero addr_lo
132 * The default value of 2, which corresponds to the only known non-zero addr_lo
133 * value used, means a single page will be sacrificed (pfn typically starts
134 * at 1). ddiphysmin can be set to 0 to disable. It cannot be set above 0x100
135 * otherwise mp startup panics.
137 pfn_t ddiphysmin
= 2;
140 check_driver_disable(void)
144 char *drv_name
, *propval
;
147 prop_name
= kmem_alloc(proplen
, KM_SLEEP
);
148 for (major
= 0; major
< devcnt
; major
++) {
149 drv_name
= ddi_major_to_name(major
);
150 if (drv_name
== NULL
)
152 (void) snprintf(prop_name
, proplen
, "disable-%s", drv_name
);
153 if (ddi_prop_lookup_string(DDI_DEV_T_ANY
, ddi_root_node(),
154 DDI_PROP_DONTPASS
, prop_name
, &propval
) == DDI_SUCCESS
) {
155 if (strcmp(propval
, "true") == 0) {
156 devnamesp
[major
].dn_flags
|= DN_DRIVER_REMOVED
;
157 cmn_err(CE_NOTE
, "driver %s disabled",
160 ddi_prop_free(propval
);
163 kmem_free(prop_name
, proplen
);
168 * Configure the hardware on the system.
169 * Called before the rootfs is mounted
174 extern void i_ddi_init_root();
177 extern int fpu_pentium_fdivbug
;
179 extern int fpu_ignored
;
182 * Determine if an FPU is attached
188 if (fpu_pentium_fdivbug
) {
190 FP hardware exhibits Pentium floating point divide problem\n");
195 printf("FP hardware will not be used\n");
196 } else if (!fpu_exists
) {
197 printf("No FPU in configuration\n");
201 * Initialize devices on the machine.
202 * Uses configuration tree built by the PROMs to determine what
203 * is present, and builds a tree of prototype dev_info nodes
204 * corresponding to the hardware which identified itself.
208 * Initialize root node.
212 /* reprogram devices not set up by firmware (BIOS) */
217 * Setup but don't startup the IOMMU
218 * Startup happens later via a direct call
219 * to IOMMU code by boot code.
220 * At this point, all PCI bus renumbering
221 * is done, so safe to init the IMMU
228 * attach the isa nexus to get ACPI resource usage
229 * isa is "kind of" a pseudo node
232 (void) i_ddi_attach_pseudo_node("isa");
234 (void) i_ddi_attach_hw_nodes("isa");
238 * The "status" property indicates the operational status of a device.
239 * If this property is present, the value is a string indicating the
240 * status of the device as follows:
242 * "okay" operational.
243 * "disabled" not operational, but might become operational.
244 * "fail" not operational because a fault has been detected,
245 * and it is unlikely that the device will become
246 * operational without repair. no additional details
248 * "fail-xxx" not operational because a fault has been detected,
249 * and it is unlikely that the device will become
250 * operational without repair. "xxx" is additional
251 * human-readable information about the particular
252 * fault condition that was detected.
254 * The absence of this property means that the operational status is
257 * This routine checks the status property of the specified device node
258 * and returns 0 if the operational status indicates failure, and 1 otherwise.
260 * The property may exist on plug-in cards the existed before IEEE 1275-1994.
261 * And, in that case, the property may not even be a string. So we carefully
262 * check for the value "fail", in the beginning of the string, noting
263 * the property length.
266 status_okay(int id
, char *buf
, int buflen
)
268 char status_buf
[OBP_MAXPROPNAME
];
272 static const char *status
= "status";
273 static const char *fail
= "fail";
274 int fail_len
= (int)strlen(fail
);
277 * Get the proplen ... if it's smaller than "fail",
278 * or doesn't exist ... then we don't care, since
279 * the value can't begin with the char string "fail".
281 * NB: proplen, if it's a string, includes the NULL in the
282 * the size of the property, and fail_len does not.
284 proplen
= prom_getproplen((pnode_t
)id
, (caddr_t
)status
);
285 if (proplen
<= fail_len
) /* nonexistant or uninteresting len */
289 * if a buffer was provided, use it
291 if ((buf
== NULL
) || (buflen
<= 0)) {
293 len
= sizeof (status_buf
);
298 * Get the property into the buffer, to the extent of the buffer,
299 * and in case the buffer is smaller than the property size,
300 * NULL terminate the buffer. (This handles the case where
301 * a buffer was passed in and the caller wants to print the
302 * value, but the buffer was too small).
304 (void) prom_bounded_getprop((pnode_t
)id
, (caddr_t
)status
,
306 *(bufp
+ len
- 1) = '\0';
309 * If the value begins with the char string "fail",
310 * then it means the node is failed. We don't care
311 * about any other values. We assume the node is ok
312 * although it might be 'disabled'.
314 if (strncmp(bufp
, fail
, fail_len
) == 0)
321 * Check the status of the device node passed as an argument.
323 * if ((status is OKAY) || (status is DISABLED))
326 * print a warning and return DDI_FAILURE
330 check_status(int id
, char *name
, dev_info_t
*parent
)
333 char devtype_buf
[OBP_MAXPROPNAME
];
334 int retval
= DDI_FAILURE
;
337 * is the status okay?
339 if (status_okay(id
, status_buf
, sizeof (status_buf
)))
340 return (DDI_SUCCESS
);
343 * a status property indicating bad memory will be associated
344 * with a node which has a "device_type" property with a value of
345 * "memory-controller". in this situation, return DDI_SUCCESS
347 if (getlongprop_buf(id
, OBP_DEVICETYPE
, devtype_buf
,
348 sizeof (devtype_buf
)) > 0) {
349 if (strcmp(devtype_buf
, "memory-controller") == 0)
350 retval
= DDI_SUCCESS
;
354 * print the status property information
356 cmn_err(CE_WARN
, "status '%s' for '%s'", status_buf
, name
);
362 softlevel1(caddr_t arg1
, caddr_t arg2
)
369 * Allow for implementation specific correction of PROM property values.
374 impl_fix_props(dev_info_t
*dip
, dev_info_t
*ch_dip
, char *name
, int len
,
378 * There are no adjustments needed in this implementation.
383 getlongprop_buf(int id
, char *name
, char *buf
, int maxlen
)
387 size
= prom_getproplen((pnode_t
)id
, name
);
388 if (size
<= 0 || (size
> maxlen
- 1))
391 if (-1 == prom_getprop((pnode_t
)id
, name
, buf
))
394 if (strcmp("name", name
) == 0) {
395 if (buf
[size
- 1] != '\0') {
405 get_prop_int_array(dev_info_t
*di
, char *pname
, int **pval
, uint_t
*plen
)
409 if ((ret
= ddi_prop_lookup_int_array(DDI_DEV_T_ANY
, di
,
410 DDI_PROP_DONTPASS
, pname
, pval
, plen
))
411 == DDI_PROP_SUCCESS
) {
412 *plen
= (*plen
) * (sizeof (int));
427 * For the x86, we're prepared to claim that the interrupt string
428 * is in the form of a list of <ipl,vec> specifications.
435 impl_xlate_intrs(dev_info_t
*child
, int *in
,
436 struct ddi_parent_private_data
*pdptr
)
440 struct intrspec
*new;
444 extern int ignore_hardware_nodes
; /* force flag from ddi_impl.c */
446 static char bad_intr_fmt
[] =
447 "bad interrupt spec from %s%d - ipl %d, irq %d\n";
450 * determine if the driver is expecting the new style "interrupts"
451 * property which just contains the IRQ, or the old style which
452 * contains pairs of <IPL,IRQ>. if it is the new style, we always
453 * assign IPL 5 unless an "interrupt-priorities" property exists.
454 * in that case, the "interrupt-priorities" property contains the
455 * IPL values that match, one for one, the IRQ values in the
456 * "interrupts" property.
459 if ((ddi_getprop(DDI_DEV_T_ANY
, child
, DDI_PROP_DONTPASS
,
460 "ignore-hardware-nodes", -1) != -1) || ignore_hardware_nodes
) {
461 /* the old style "interrupts" property... */
464 * The list consists of <ipl,vec> elements
466 if ((n
= (*in
++ >> 1)) < 1)
467 return (DDI_FAILURE
);
469 pdptr
->par_nintr
= n
;
470 size
= n
* sizeof (struct intrspec
);
471 new = pdptr
->par_intr
= kmem_zalloc(size
, KM_SLEEP
);
477 if (level
< 1 || level
> MAXIPL
||
478 vec
< VEC_MIN
|| vec
> VEC_MAX
) {
479 cmn_err(CE_CONT
, bad_intr_fmt
,
480 DEVI(child
)->devi_name
,
481 DEVI(child
)->devi_instance
, level
, vec
);
484 new->intrspec_pri
= level
;
486 new->intrspec_vec
= vec
;
489 * irq 2 on the PC bus is tied to irq 9
490 * on ISA, EISA and MicroChannel
492 new->intrspec_vec
= 9;
496 return (DDI_SUCCESS
);
498 /* the new style "interrupts" property... */
501 * The list consists of <vec> elements
503 if ((n
= (*in
++)) < 1)
504 return (DDI_FAILURE
);
506 pdptr
->par_nintr
= n
;
507 size
= n
* sizeof (struct intrspec
);
508 new = pdptr
->par_intr
= kmem_zalloc(size
, KM_SLEEP
);
510 /* XXX check for "interrupt-priorities" property... */
511 if (ddi_getlongprop(DDI_DEV_T_ANY
, child
, DDI_PROP_DONTPASS
,
512 "interrupt-priorities", (caddr_t
)&got_prop
, &got_len
)
513 == DDI_PROP_SUCCESS
) {
514 if (n
!= (got_len
/ sizeof (int))) {
516 "bad interrupt-priorities length"
517 " from %s%d: expected %d, got %d\n",
518 DEVI(child
)->devi_name
,
519 DEVI(child
)->devi_instance
, n
,
520 (int)(got_len
/ sizeof (int)));
523 inpri
= (int *)got_prop
;
535 if (level
< 1 || level
> MAXIPL
||
536 vec
< VEC_MIN
|| vec
> VEC_MAX
) {
537 cmn_err(CE_CONT
, bad_intr_fmt
,
538 DEVI(child
)->devi_name
,
539 DEVI(child
)->devi_instance
, level
, vec
);
542 new->intrspec_pri
= level
;
544 new->intrspec_vec
= vec
;
547 * irq 2 on the PC bus is tied to irq 9
548 * on ISA, EISA and MicroChannel
550 new->intrspec_vec
= 9;
555 kmem_free(got_prop
, got_len
);
556 return (DDI_SUCCESS
);
560 kmem_free(pdptr
->par_intr
, size
);
561 pdptr
->par_intr
= NULL
;
562 pdptr
->par_nintr
= 0;
564 kmem_free(got_prop
, got_len
);
566 return (DDI_FAILURE
);
570 * Create a ddi_parent_private_data structure from the ddi properties of
573 * The "reg" and either an "intr" or "interrupts" properties are required
574 * if the driver wishes to create mappings or field interrupts on behalf
577 * The "reg" property is assumed to be a list of at least one triple
579 * <bustype, address, size>*1
581 * The "intr" property is assumed to be a list of at least one duple
583 * <SPARC ipl, vector#>*1
585 * The "interrupts" property is assumed to be a list of at least one
586 * n-tuples that describes the interrupt capabilities of the bus the device
587 * is connected to. For SBus, this looks like
591 * (This property obsoletes the 'intr' property).
593 * The "ranges" property is optional.
596 make_ddi_ppd(dev_info_t
*child
, struct ddi_parent_private_data
**ppd
)
598 struct ddi_parent_private_data
*pdptr
;
600 int *reg_prop
, *rng_prop
, *intr_prop
, *irupts_prop
;
601 uint_t reg_len
, rng_len
, intr_len
, irupts_len
;
603 *ppd
= pdptr
= kmem_zalloc(sizeof (*pdptr
), KM_SLEEP
);
606 * Handle the 'reg' property.
608 if ((get_prop_int_array(child
, "reg", ®_prop
, ®_len
) ==
609 DDI_PROP_SUCCESS
) && (reg_len
!= 0)) {
610 pdptr
->par_nreg
= reg_len
/ (int)sizeof (struct regspec
);
611 pdptr
->par_reg
= (struct regspec
*)reg_prop
;
615 * See if I have a range (adding one where needed - this
616 * means to add one for sbus node in sun4c, when romvec > 0,
617 * if no range is already defined in the PROM node.
618 * (Currently no sun4c PROMS define range properties,
619 * but they should and may in the future.) For the SBus
620 * node, the range is defined by the SBus reg property.
622 if (get_prop_int_array(child
, "ranges", &rng_prop
, &rng_len
)
623 == DDI_PROP_SUCCESS
) {
624 pdptr
->par_nrng
= rng_len
/ (int)(sizeof (struct rangespec
));
625 pdptr
->par_rng
= (struct rangespec
*)rng_prop
;
629 * Handle the 'intr' and 'interrupts' properties
633 * For backwards compatibility
634 * we first look for the 'intr' property for the device.
636 if (get_prop_int_array(child
, "intr", &intr_prop
, &intr_len
)
637 != DDI_PROP_SUCCESS
) {
642 * If we're to support bus adapters and future platforms cleanly,
643 * we need to support the generalized 'interrupts' property.
645 if (get_prop_int_array(child
, "interrupts", &irupts_prop
,
646 &irupts_len
) != DDI_PROP_SUCCESS
) {
648 } else if (intr_len
!= 0) {
650 * If both 'intr' and 'interrupts' are defined,
651 * then 'interrupts' wins and we toss the 'intr' away.
653 ddi_prop_free((void *)intr_prop
);
660 * Translate the 'intr' property into an array
661 * an array of struct intrspec's. There's not really
662 * very much to do here except copy what's out there.
665 struct intrspec
*new;
666 struct prop_ispec
*l
;
668 n
= pdptr
->par_nintr
= intr_len
/ sizeof (struct prop_ispec
);
669 l
= (struct prop_ispec
*)intr_prop
;
671 new = kmem_zalloc(n
* sizeof (struct intrspec
), KM_SLEEP
);
673 new->intrspec_pri
= l
->pri
;
674 new->intrspec_vec
= l
->vec
;
678 ddi_prop_free((void *)intr_prop
);
680 } else if ((n
= irupts_len
) != 0) {
685 * Translate the 'interrupts' property into an array
686 * of intrspecs for the rest of the DDI framework to
687 * toy with. Only our ancestors really know how to
688 * do this, so ask 'em. We massage the 'interrupts'
689 * property so that it is pre-pended by a count of
690 * the number of integers in the argument.
692 size
= sizeof (int) + n
;
693 out
= kmem_alloc(size
, KM_SLEEP
);
694 *out
= n
/ sizeof (int);
695 bcopy(irupts_prop
, out
+ 1, (size_t)n
);
696 ddi_prop_free((void *)irupts_prop
);
697 if (impl_xlate_intrs(child
, out
, pdptr
) != DDI_SUCCESS
) {
699 "Unable to translate 'interrupts' for %s%d\n",
700 DEVI(child
)->devi_binding_name
,
701 DEVI(child
)->devi_instance
);
703 kmem_free(out
, size
);
711 impl_sunbus_name_child(dev_info_t
*child
, char *name
, int namelen
)
714 * Fill in parent-private data and this function returns to us
715 * an indication if it used "registers" to fill in the data.
717 if (ddi_get_parent_data(child
) == NULL
) {
718 struct ddi_parent_private_data
*pdptr
;
719 make_ddi_ppd(child
, &pdptr
);
720 ddi_set_parent_data(child
, pdptr
);
724 if (sparc_pd_getnreg(child
) > 0) {
725 (void) snprintf(name
, namelen
, "%x,%x",
726 (uint_t
)sparc_pd_getreg(child
, 0)->regspec_bustype
,
727 (uint_t
)sparc_pd_getreg(child
, 0)->regspec_addr
);
730 return (DDI_SUCCESS
);
734 * Called from the bus_ctl op of sunbus (sbus, obio, etc) nexus drivers
735 * to implement the DDI_CTLOPS_INITCHILD operation. That is, it names
736 * the children of sun busses based on the reg spec.
738 * Handles the following properties (in make_ddi_ppd):
742 * intr old-form interrupt spec
743 * interrupts new (bus-oriented) interrupt spec
747 impl_ddi_sunbus_initchild(dev_info_t
*child
)
749 char name
[MAXNAMELEN
];
750 void impl_ddi_sunbus_removechild(dev_info_t
*);
753 * Name the child, also makes parent private data
755 (void) impl_sunbus_name_child(child
, name
, MAXNAMELEN
);
756 ddi_set_name_addr(child
, name
);
759 * Attempt to merge a .conf node; if successful, remove the
762 if ((ndi_dev_is_persistent_node(child
) == 0) &&
763 (ndi_merge_node(child
, impl_sunbus_name_child
) == DDI_SUCCESS
)) {
765 * Return failure to remove node
767 impl_ddi_sunbus_removechild(child
);
768 return (DDI_FAILURE
);
770 return (DDI_SUCCESS
);
774 impl_free_ddi_ppd(dev_info_t
*dip
)
776 struct ddi_parent_private_data
*pdptr
;
779 if ((pdptr
= ddi_get_parent_data(dip
)) == NULL
)
782 if ((n
= (size_t)pdptr
->par_nintr
) != 0)
784 * Note that kmem_free is used here (instead of
785 * ddi_prop_free) because the contents of the
786 * property were placed into a separate buffer and
787 * mucked with a bit before being stored in par_intr.
788 * The actual return value from the prop lookup
789 * was freed with ddi_prop_free previously.
791 kmem_free(pdptr
->par_intr
, n
* sizeof (struct intrspec
));
793 if ((n
= (size_t)pdptr
->par_nrng
) != 0)
794 ddi_prop_free((void *)pdptr
->par_rng
);
796 if ((n
= pdptr
->par_nreg
) != 0)
797 ddi_prop_free((void *)pdptr
->par_reg
);
799 kmem_free(pdptr
, sizeof (*pdptr
));
800 ddi_set_parent_data(dip
, NULL
);
804 impl_ddi_sunbus_removechild(dev_info_t
*dip
)
806 impl_free_ddi_ppd(dip
);
807 ddi_set_name_addr(dip
, NULL
);
809 * Strip the node to properly convert it back to prototype form
811 impl_rem_dev_props(dip
);
819 * turn this on to force isa, eisa, and mca device to ignore the new
820 * hardware nodes in the device tree (normally turned on only for
821 * drivers that need it by setting the property "ignore-hardware-nodes"
822 * in their driver.conf file).
824 * 7/31/96 -- Turned off globally. Leaving variable in for the moment
827 int ignore_hardware_nodes
= 0;
832 static struct impl_bus_promops
*impl_busp
;
836 * New DDI interrupt framework
842 * This is the interrupt operator function wrapper for the bus function
846 i_ddi_intr_ops(dev_info_t
*dip
, dev_info_t
*rdip
, ddi_intr_op_t op
,
847 ddi_intr_handle_impl_t
*hdlp
, void * result
)
849 dev_info_t
*pdip
= (dev_info_t
*)DEVI(dip
)->devi_parent
;
850 int ret
= DDI_FAILURE
;
852 /* request parent to process this interrupt op */
853 if (NEXUS_HAS_INTR_OP(pdip
))
854 ret
= (*(DEVI(pdip
)->devi_ops
->devo_bus_ops
->bus_intr_op
))(
855 pdip
, rdip
, op
, hdlp
, result
);
857 cmn_err(CE_WARN
, "Failed to process interrupt "
858 "for %s%d due to down-rev nexus driver %s%d",
859 ddi_get_name(rdip
), ddi_get_instance(rdip
),
860 ddi_get_name(pdip
), ddi_get_instance(pdip
));
865 * i_ddi_add_softint - allocate and add a soft interrupt to the system
868 i_ddi_add_softint(ddi_softint_hdl_impl_t
*hdlp
)
872 /* add soft interrupt handler */
873 ret
= add_avsoftintr((void *)hdlp
, hdlp
->ih_pri
, hdlp
->ih_cb_func
,
874 DEVI(hdlp
->ih_dip
)->devi_name
, hdlp
->ih_cb_arg1
, hdlp
->ih_cb_arg2
);
875 return (ret
? DDI_SUCCESS
: DDI_FAILURE
);
880 i_ddi_remove_softint(ddi_softint_hdl_impl_t
*hdlp
)
882 (void) rem_avsoftintr((void *)hdlp
, hdlp
->ih_pri
, hdlp
->ih_cb_func
);
886 extern void (*setsoftint
)(int, struct av_softinfo
*);
887 extern boolean_t
av_check_softint_pending(struct av_softinfo
*, boolean_t
);
890 i_ddi_trigger_softint(ddi_softint_hdl_impl_t
*hdlp
, void *arg2
)
892 if (av_check_softint_pending(hdlp
->ih_pending
, B_FALSE
))
893 return (DDI_EPENDING
);
895 update_avsoftintr_args((void *)hdlp
, hdlp
->ih_pri
, arg2
);
897 (*setsoftint
)(hdlp
->ih_pri
, hdlp
->ih_pending
);
898 return (DDI_SUCCESS
);
902 * i_ddi_set_softint_pri:
904 * The way this works is that it first tries to add a softint vector
905 * at the new priority in hdlp. If that succeeds; then it removes the
906 * existing softint vector at the old priority.
909 i_ddi_set_softint_pri(ddi_softint_hdl_impl_t
*hdlp
, uint_t old_pri
)
914 * If a softint is pending at the old priority then fail the request.
916 if (av_check_softint_pending(hdlp
->ih_pending
, B_TRUE
))
917 return (DDI_FAILURE
);
919 ret
= av_softint_movepri((void *)hdlp
, old_pri
);
920 return (ret
? DDI_SUCCESS
: DDI_FAILURE
);
924 i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t
*hdlp
)
926 hdlp
->ih_private
= (void *)kmem_zalloc(sizeof (ihdl_plat_t
), KM_SLEEP
);
930 i_ddi_free_intr_phdl(ddi_intr_handle_impl_t
*hdlp
)
932 kmem_free(hdlp
->ih_private
, sizeof (ihdl_plat_t
));
933 hdlp
->ih_private
= NULL
;
937 i_ddi_get_intx_nintrs(dev_info_t
*dip
)
939 struct ddi_parent_private_data
*pdp
;
941 if ((pdp
= ddi_get_parent_data(dip
)) == NULL
)
944 return (pdp
->par_nintr
);
952 * Support for allocating DMAable memory to implement
953 * ddi_dma_mem_alloc(9F) interface.
956 #define KA_ALIGN_SHIFT 7
957 #define KA_ALIGN (1 << KA_ALIGN_SHIFT)
958 #define KA_NCACHE (PAGESHIFT + 1 - KA_ALIGN_SHIFT)
961 * Dummy DMA attribute template for kmem_io[].kmem_io_attr. We only
962 * care about addr_lo, addr_hi, and align. addr_hi will be dynamically set.
965 static ddi_dma_attr_t kmem_io_attr
= {
967 0x0000000000000000ULL
, /* dma_attr_addr_lo */
968 0x0000000000000000ULL
, /* dma_attr_addr_hi */
970 0x1000, /* dma_attr_align */
971 1, 1, 0xffffffffULL
, 0xffffffffULL
, 0x1, 1, 0
974 /* kmem io memory ranges and indices */
976 IO_4P
, IO_64G
, IO_4G
, IO_2G
, IO_1G
, IO_512M
,
977 IO_256M
, IO_128M
, IO_64M
, IO_32M
, IO_16M
, MAX_MEM_RANGES
981 vmem_t
*kmem_io_arena
;
982 kmem_cache_t
*kmem_io_cache
[KA_NCACHE
];
983 ddi_dma_attr_t kmem_io_attr
;
984 } kmem_io
[MAX_MEM_RANGES
];
986 static int kmem_io_idx
; /* index of first populated kmem_io[] */
989 page_create_io_wrapper(void *addr
, size_t len
, int vmflag
, void *arg
)
991 extern page_t
*page_create_io(vnode_t
*, uoff_t
, uint_t
,
992 uint_t
, struct as
*, caddr_t
, ddi_dma_attr_t
*);
994 return (page_create_io(&kvp
, (uoff_t
)(uintptr_t)addr
, len
,
995 PG_EXCL
| ((vmflag
& VM_NOSLEEP
) ? 0 : PG_WAIT
), &kas
, addr
, arg
));
999 segkmem_alloc_io_4P(vmem_t
*vmp
, size_t size
, int vmflag
)
1001 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1002 page_create_io_wrapper
, &kmem_io
[IO_4P
].kmem_io_attr
));
1006 segkmem_alloc_io_64G(vmem_t
*vmp
, size_t size
, int vmflag
)
1008 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1009 page_create_io_wrapper
, &kmem_io
[IO_64G
].kmem_io_attr
));
1013 segkmem_alloc_io_4G(vmem_t
*vmp
, size_t size
, int vmflag
)
1015 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1016 page_create_io_wrapper
, &kmem_io
[IO_4G
].kmem_io_attr
));
1020 segkmem_alloc_io_2G(vmem_t
*vmp
, size_t size
, int vmflag
)
1022 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1023 page_create_io_wrapper
, &kmem_io
[IO_2G
].kmem_io_attr
));
1027 segkmem_alloc_io_1G(vmem_t
*vmp
, size_t size
, int vmflag
)
1029 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1030 page_create_io_wrapper
, &kmem_io
[IO_1G
].kmem_io_attr
));
1034 segkmem_alloc_io_512M(vmem_t
*vmp
, size_t size
, int vmflag
)
1036 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1037 page_create_io_wrapper
, &kmem_io
[IO_512M
].kmem_io_attr
));
1041 segkmem_alloc_io_256M(vmem_t
*vmp
, size_t size
, int vmflag
)
1043 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1044 page_create_io_wrapper
, &kmem_io
[IO_256M
].kmem_io_attr
));
1048 segkmem_alloc_io_128M(vmem_t
*vmp
, size_t size
, int vmflag
)
1050 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1051 page_create_io_wrapper
, &kmem_io
[IO_128M
].kmem_io_attr
));
1055 segkmem_alloc_io_64M(vmem_t
*vmp
, size_t size
, int vmflag
)
1057 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1058 page_create_io_wrapper
, &kmem_io
[IO_64M
].kmem_io_attr
));
1062 segkmem_alloc_io_32M(vmem_t
*vmp
, size_t size
, int vmflag
)
1064 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1065 page_create_io_wrapper
, &kmem_io
[IO_32M
].kmem_io_attr
));
1069 segkmem_alloc_io_16M(vmem_t
*vmp
, size_t size
, int vmflag
)
1071 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
1072 page_create_io_wrapper
, &kmem_io
[IO_16M
].kmem_io_attr
));
1078 void *(*io_alloc
)(vmem_t
*, size_t, int);
1079 int io_initial
; /* kmem_io_init during startup */
1080 } io_arena_params
[MAX_MEM_RANGES
] = {
1081 {0x000fffffffffffffULL
, "kmem_io_4P", segkmem_alloc_io_4P
, 1},
1082 {0x0000000fffffffffULL
, "kmem_io_64G", segkmem_alloc_io_64G
, 0},
1083 {0x00000000ffffffffULL
, "kmem_io_4G", segkmem_alloc_io_4G
, 1},
1084 {0x000000007fffffffULL
, "kmem_io_2G", segkmem_alloc_io_2G
, 1},
1085 {0x000000003fffffffULL
, "kmem_io_1G", segkmem_alloc_io_1G
, 0},
1086 {0x000000001fffffffULL
, "kmem_io_512M", segkmem_alloc_io_512M
, 0},
1087 {0x000000000fffffffULL
, "kmem_io_256M", segkmem_alloc_io_256M
, 0},
1088 {0x0000000007ffffffULL
, "kmem_io_128M", segkmem_alloc_io_128M
, 0},
1089 {0x0000000003ffffffULL
, "kmem_io_64M", segkmem_alloc_io_64M
, 0},
1090 {0x0000000001ffffffULL
, "kmem_io_32M", segkmem_alloc_io_32M
, 0},
1091 {0x0000000000ffffffULL
, "kmem_io_16M", segkmem_alloc_io_16M
, 1}
1100 kmem_io
[a
].kmem_io_arena
= vmem_create(io_arena_params
[a
].io_name
,
1101 NULL
, 0, PAGESIZE
, io_arena_params
[a
].io_alloc
,
1103 heap_arena
, 0, VM_SLEEP
);
1105 for (c
= 0; c
< KA_NCACHE
; c
++) {
1106 size_t size
= KA_ALIGN
<< c
;
1107 (void) sprintf(name
, "%s_%lu",
1108 io_arena_params
[a
].io_name
, size
);
1109 kmem_io
[a
].kmem_io_cache
[c
] = kmem_cache_create(name
,
1110 size
, size
, NULL
, NULL
, NULL
, NULL
,
1111 kmem_io
[a
].kmem_io_arena
, 0);
1116 * Return the index of the highest memory range for addr.
1119 kmem_io_index(uint64_t addr
)
1123 for (n
= kmem_io_idx
; n
< MAX_MEM_RANGES
; n
++) {
1124 if (kmem_io
[n
].kmem_io_attr
.dma_attr_addr_hi
<= addr
) {
1125 if (kmem_io
[n
].kmem_io_arena
== NULL
)
1130 panic("kmem_io_index: invalid addr - must be at least 16m");
1136 * Return the index of the next kmem_io populated memory range
1140 kmem_io_index_next(int curindex
)
1144 for (n
= curindex
+ 1; n
< MAX_MEM_RANGES
; n
++) {
1145 if (kmem_io
[n
].kmem_io_arena
)
1152 * allow kmem to be mapped in with different PTE cache attribute settings.
1153 * Used by i_ddi_mem_alloc()
1156 kmem_override_cache_attrs(caddr_t kva
, size_t size
, uint_t order
)
1163 if (hat_getattr(kas
.a_hat
, kva
, &hat_attr
) == -1) {
1167 hat_attr
&= ~HAT_ORDER_MASK
;
1168 hat_attr
|= order
| HAT_NOSYNC
;
1169 hat_flags
= HAT_LOAD_LOCK
;
1171 kva_end
= (caddr_t
)(((uintptr_t)kva
+ size
+ PAGEOFFSET
) &
1172 (uintptr_t)PAGEMASK
);
1173 kva
= (caddr_t
)((uintptr_t)kva
& (uintptr_t)PAGEMASK
);
1175 while (kva
< kva_end
) {
1176 pfn
= hat_getpfnum(kas
.a_hat
, kva
);
1177 hat_unload(kas
.a_hat
, kva
, PAGESIZE
, HAT_UNLOAD_UNLOCK
);
1178 hat_devload(kas
.a_hat
, kva
, PAGESIZE
, pfn
, hat_attr
, hat_flags
);
1179 kva
+= MMU_PAGESIZE
;
1186 ctgcompare(const void *a1
, const void *a2
)
1188 /* we just want to compare virtual addresses */
1189 a1
= ((struct ctgas
*)a1
)->ctg_addr
;
1190 a2
= ((struct ctgas
*)a2
)->ctg_addr
;
1191 return (a1
== a2
? 0 : (a1
< a2
? -1 : 1));
1198 paddr_t maxphysaddr
;
1199 extern pfn_t physmax
;
1201 maxphysaddr
= mmu_ptob((paddr_t
)physmax
) + MMU_PAGEOFFSET
;
1203 ASSERT(maxphysaddr
<= io_arena_params
[0].io_limit
);
1205 for (a
= 0; a
< MAX_MEM_RANGES
; a
++) {
1206 if (maxphysaddr
>= io_arena_params
[a
+ 1].io_limit
) {
1207 if (maxphysaddr
> io_arena_params
[a
+ 1].io_limit
)
1208 io_arena_params
[a
].io_limit
= maxphysaddr
;
1216 for (; a
< MAX_MEM_RANGES
; a
++) {
1217 kmem_io
[a
].kmem_io_attr
= kmem_io_attr
;
1218 kmem_io
[a
].kmem_io_attr
.dma_attr_addr_hi
=
1219 io_arena_params
[a
].io_limit
;
1221 * initialize kmem_io[] arena/cache corresponding to
1222 * maxphysaddr and to the "common" io memory ranges that
1223 * have io_initial set to a non-zero value.
1225 if (io_arena_params
[a
].io_initial
|| a
== kmem_io_idx
)
1229 /* initialize ctgtree */
1230 avl_create(&ctgtree
, ctgcompare
, sizeof (struct ctgas
),
1231 offsetof(struct ctgas
, ctg_link
));
1235 * put contig address/size
1238 putctgas(void *addr
, size_t size
)
1241 if ((ctgp
= kmem_zalloc(sizeof (*ctgp
), KM_NOSLEEP
)) != NULL
) {
1242 ctgp
->ctg_addr
= addr
;
1243 ctgp
->ctg_size
= size
;
1245 avl_add(&ctgtree
, ctgp
);
1252 * get contig size by addr
1255 getctgsz(void *addr
)
1261 find
.ctg_addr
= addr
;
1263 if ((ctgp
= avl_find(&ctgtree
, &find
, NULL
)) != NULL
) {
1264 avl_remove(&ctgtree
, ctgp
);
1269 sz
= ctgp
->ctg_size
;
1270 kmem_free(ctgp
, sizeof (*ctgp
));
1279 * allocates contiguous memory to satisfy the 'size' and dma attributes
1280 * specified in 'attr'.
1282 * Not all of memory need to be physically contiguous if the
1283 * scatter-gather list length is greater than 1.
1288 contig_alloc(size_t size
, ddi_dma_attr_t
*attr
, uintptr_t align
, int cansleep
)
1290 pgcnt_t pgcnt
= btopr(size
);
1291 size_t asize
= pgcnt
* PAGESIZE
;
1296 extern page_t
*page_create_io(vnode_t
*, uoff_t
, uint_t
,
1297 uint_t
, struct as
*, caddr_t
, ddi_dma_attr_t
*);
1299 /* segkmem_xalloc */
1301 if (align
<= PAGESIZE
)
1302 addr
= vmem_alloc(heap_arena
, asize
,
1303 (cansleep
) ? VM_SLEEP
: VM_NOSLEEP
);
1305 addr
= vmem_xalloc(heap_arena
, asize
, align
, 0, 0, NULL
, NULL
,
1306 (cansleep
) ? VM_SLEEP
: VM_NOSLEEP
);
1308 ASSERT(!((uintptr_t)addr
& (align
- 1)));
1310 if (page_resv(pgcnt
, (cansleep
) ? KM_SLEEP
: KM_NOSLEEP
) == 0) {
1311 vmem_free(heap_arena
, addr
, asize
);
1319 /* 4k req gets from freelists rather than pfn search */
1320 if (pgcnt
> 1 || align
> PAGESIZE
)
1321 pflag
|= PG_PHYSCONTIG
;
1323 ppl
= page_create_io(&kvp
, (uoff_t
)(uintptr_t)addr
,
1324 asize
, pflag
, &kas
, (caddr_t
)addr
, attr
);
1327 vmem_free(heap_arena
, addr
, asize
);
1332 while (ppl
!= NULL
) {
1335 ASSERT(page_iolock_assert(pp
));
1338 hat_memload(kas
.a_hat
, (caddr_t
)(uintptr_t)pp
->p_offset
,
1339 pp
, (PROT_ALL
& ~PROT_USER
) |
1340 HAT_NOSYNC
, HAT_LOAD_LOCK
);
1347 contig_free(void *addr
, size_t size
)
1349 pgcnt_t pgcnt
= btopr(size
);
1350 size_t asize
= pgcnt
* PAGESIZE
;
1354 hat_unload(kas
.a_hat
, addr
, asize
, HAT_UNLOAD_UNLOCK
);
1356 for (a
= addr
, ea
= a
+ asize
; a
< ea
; a
+= PAGESIZE
) {
1357 pp
= page_find(&kvp
.v_object
, (uoff_t
)(uintptr_t)a
);
1359 panic("contig_free: contig pp not found");
1361 if (!page_tryupgrade(pp
)) {
1363 pp
= page_lookup(&kvp
.v_object
,
1364 (uoff_t
)(uintptr_t)a
, SE_EXCL
);
1366 panic("contig_free: page freed");
1368 page_destroy(pp
, 0);
1372 vmem_free(heap_arena
, addr
, asize
);
1376 * Allocate from the system, aligned on a specific boundary.
1377 * The alignment, if non-zero, must be a power of 2.
1380 kalloca(size_t size
, size_t align
, int cansleep
, int physcontig
,
1381 ddi_dma_attr_t
*attr
)
1383 size_t *addr
, *raddr
, rsize
;
1384 size_t hdrsize
= 4 * sizeof (size_t); /* must be power of 2 */
1387 kmem_cache_t
*cp
= NULL
;
1389 if (attr
->dma_attr_addr_lo
> mmu_ptob((uint64_t)ddiphysmin
))
1392 align
= MAX(align
, hdrsize
);
1393 ASSERT((align
& (align
- 1)) == 0);
1396 * All of our allocators guarantee 16-byte alignment, so we don't
1397 * need to reserve additional space for the header.
1398 * To simplify picking the correct kmem_io_cache, we round up to
1399 * a multiple of KA_ALIGN.
1401 rsize
= P2ROUNDUP_TYPED(size
+ align
, KA_ALIGN
, size_t);
1403 if (physcontig
&& rsize
> PAGESIZE
) {
1404 if (addr
= contig_alloc(size
, attr
, align
, cansleep
)) {
1405 if (!putctgas(addr
, size
))
1406 contig_free(addr
, size
);
1413 a
= kmem_io_index(attr
->dma_attr_addr_hi
);
1415 if (rsize
> PAGESIZE
) {
1416 vmp
= kmem_io
[a
].kmem_io_arena
;
1417 raddr
= vmem_alloc(vmp
, rsize
,
1418 (cansleep
) ? VM_SLEEP
: VM_NOSLEEP
);
1420 c
= highbit((rsize
>> KA_ALIGN_SHIFT
) - 1);
1421 cp
= kmem_io
[a
].kmem_io_cache
[c
];
1422 raddr
= kmem_cache_alloc(cp
, (cansleep
) ? KM_SLEEP
:
1426 if (raddr
== NULL
) {
1429 ASSERT(cansleep
== 0);
1430 if (rsize
> PAGESIZE
)
1433 * System does not have memory in the requested range.
1434 * Try smaller kmem io ranges and larger cache sizes
1435 * to see if there might be memory available in
1436 * these other caches.
1439 for (na
= kmem_io_index_next(a
); na
>= 0;
1440 na
= kmem_io_index_next(na
)) {
1441 ASSERT(kmem_io
[na
].kmem_io_arena
);
1442 cp
= kmem_io
[na
].kmem_io_cache
[c
];
1443 raddr
= kmem_cache_alloc(cp
, KM_NOSLEEP
);
1447 /* now try the larger kmem io cache sizes */
1448 for (na
= a
; na
>= 0; na
= kmem_io_index_next(na
)) {
1449 for (i
= c
+ 1; i
< KA_NCACHE
; i
++) {
1450 cp
= kmem_io
[na
].kmem_io_cache
[i
];
1451 raddr
= kmem_cache_alloc(cp
, KM_NOSLEEP
);
1460 ASSERT(!P2BOUNDARY((uintptr_t)raddr
, rsize
, PAGESIZE
) ||
1463 addr
= (size_t *)P2ROUNDUP((uintptr_t)raddr
+ hdrsize
, align
);
1464 ASSERT((uintptr_t)addr
+ size
- (uintptr_t)raddr
<= rsize
);
1466 addr
[-4] = (size_t)cp
;
1467 addr
[-3] = (size_t)vmp
;
1468 addr
[-2] = (size_t)raddr
;
1479 if (!((uintptr_t)addr
& PAGEOFFSET
) && (size
= getctgsz(addr
))) {
1480 contig_free(addr
, size
);
1482 size_t *saddr
= addr
;
1484 vmem_free((vmem_t
*)saddr
[-3], (void *)saddr
[-2],
1487 kmem_cache_free((kmem_cache_t
*)saddr
[-4],
1494 i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t
*devaccp
, uint_t
*hataccp
)
1499 * Check if the specified cache attribute is supported on the platform.
1500 * This function must be called before i_ddi_cacheattr_to_hatacc().
1503 i_ddi_check_cache_attr(uint_t flags
)
1506 * The cache attributes are mutually exclusive. Any combination of
1507 * the attributes leads to a failure.
1509 uint_t cache_attr
= IOMEM_CACHE_ATTR(flags
);
1510 if ((cache_attr
!= 0) && !ISP2(cache_attr
))
1513 /* All cache attributes are supported on X86/X64 */
1514 if (cache_attr
& (IOMEM_DATA_UNCACHED
| IOMEM_DATA_CACHED
|
1515 IOMEM_DATA_UC_WR_COMBINE
))
1518 /* undefined attributes */
1522 /* set HAT cache attributes from the cache attributes */
1524 i_ddi_cacheattr_to_hatacc(uint_t flags
, uint_t
*hataccp
)
1526 uint_t cache_attr
= IOMEM_CACHE_ATTR(flags
);
1527 static char *fname
= "i_ddi_cacheattr_to_hatacc";
1530 * If write-combining is not supported, then it falls back
1533 if (cache_attr
== IOMEM_DATA_UC_WR_COMBINE
&&
1534 !is_x86_feature(x86_featureset
, X86FSET_PAT
))
1535 cache_attr
= IOMEM_DATA_UNCACHED
;
1538 * set HAT attrs according to the cache attrs.
1540 switch (cache_attr
) {
1541 case IOMEM_DATA_UNCACHED
:
1542 *hataccp
&= ~HAT_ORDER_MASK
;
1543 *hataccp
|= (HAT_STRICTORDER
| HAT_PLAT_NOCACHE
);
1545 case IOMEM_DATA_UC_WR_COMBINE
:
1546 *hataccp
&= ~HAT_ORDER_MASK
;
1547 *hataccp
|= (HAT_MERGING_OK
| HAT_PLAT_NOCACHE
);
1549 case IOMEM_DATA_CACHED
:
1550 *hataccp
&= ~HAT_ORDER_MASK
;
1551 *hataccp
|= HAT_UNORDERED_OK
;
1554 * This case must not occur because the cache attribute is scrutinized
1555 * before this function is called.
1559 * set cacheable to hat attrs.
1561 *hataccp
&= ~HAT_ORDER_MASK
;
1562 *hataccp
|= HAT_UNORDERED_OK
;
1563 cmn_err(CE_WARN
, "%s: cache_attr=0x%x is ignored.",
1569 * This should actually be called i_ddi_dma_mem_alloc. There should
1570 * also be an i_ddi_pio_mem_alloc. i_ddi_dma_mem_alloc should call
1571 * through the device tree with the DDI_CTLOPS_DMA_ALIGN ctl ops to
1572 * get alignment requirements for DMA memory. i_ddi_pio_mem_alloc
1573 * should use DDI_CTLOPS_PIO_ALIGN. Since we only have i_ddi_mem_alloc
1574 * so far which is used for both, DMA and PIO, we have to use the DMA
1575 * ctl ops to make everybody happy.
1579 i_ddi_mem_alloc(dev_info_t
*dip
, ddi_dma_attr_t
*attr
,
1580 size_t length
, int cansleep
, int flags
,
1581 ddi_device_acc_attr_t
*accattrp
, caddr_t
*kaddrp
,
1582 size_t *real_length
, ddi_acc_hdl_t
*ap
)
1586 ddi_acc_impl_t
*iap
;
1594 * Check legality of arguments
1596 if (length
== 0 || kaddrp
== NULL
|| attr
== NULL
) {
1597 return (DDI_FAILURE
);
1600 if (attr
->dma_attr_minxfer
== 0 || attr
->dma_attr_align
== 0 ||
1601 !ISP2(attr
->dma_attr_align
) || !ISP2(attr
->dma_attr_minxfer
)) {
1602 return (DDI_FAILURE
);
1606 * figure out most restrictive alignment requirement
1608 iomin
= attr
->dma_attr_minxfer
;
1609 iomin
= maxbit(iomin
, attr
->dma_attr_align
);
1611 return (DDI_FAILURE
);
1613 ASSERT((iomin
& (iomin
- 1)) == 0);
1616 * if we allocate memory with IOMEM_DATA_UNCACHED or
1617 * IOMEM_DATA_UC_WR_COMBINE, make sure we allocate a page aligned
1618 * memory that ends on a page boundry.
1619 * Don't want to have to different cache mappings to the same
1622 if (OVERRIDE_CACHE_ATTR(flags
)) {
1623 iomin
= (iomin
+ MMU_PAGEOFFSET
) & MMU_PAGEMASK
;
1624 length
= (length
+ MMU_PAGEOFFSET
) & (size_t)MMU_PAGEMASK
;
1628 * Determine if we need to satisfy the request for physically
1629 * contiguous memory or alignments larger than pagesize.
1631 npages
= btopr(length
+ attr
->dma_attr_align
);
1632 minctg
= howmany(npages
, attr
->dma_attr_sgllen
);
1635 uint64_t pfnseg
= attr
->dma_attr_seg
>> PAGESHIFT
;
1637 * verify that the minimum contig requirement for the
1638 * actual length does not cross segment boundary.
1640 length
= P2ROUNDUP_TYPED(length
, attr
->dma_attr_minxfer
,
1642 npages
= btopr(length
);
1643 minctg
= howmany(npages
, attr
->dma_attr_sgllen
);
1644 if (minctg
> pfnseg
+ 1)
1645 return (DDI_FAILURE
);
1648 length
= P2ROUNDUP_TYPED(length
, iomin
, size_t);
1652 * Allocate the requested amount from the system.
1654 a
= kalloca(length
, iomin
, cansleep
, physcontig
, attr
);
1656 if ((*kaddrp
= a
) == NULL
)
1657 return (DDI_FAILURE
);
1660 * if we to modify the cache attributes, go back and muck with the
1663 if (OVERRIDE_CACHE_ATTR(flags
)) {
1665 i_ddi_cacheattr_to_hatacc(flags
, &order
);
1666 e
= kmem_override_cache_attrs(a
, length
, order
);
1669 return (DDI_FAILURE
);
1674 *real_length
= length
;
1678 * initialize access handle
1680 iap
= (ddi_acc_impl_t
*)ap
->ah_platform_private
;
1681 iap
->ahi_acc_attr
|= DDI_ACCATTR_CPU_VADDR
;
1682 impl_acc_hdl_init(ap
);
1685 return (DDI_SUCCESS
);
1690 i_ddi_mem_free(caddr_t kaddr
, ddi_acc_hdl_t
*ap
)
1694 * if we modified the cache attributes on alloc, go back and
1695 * fix them since this memory could be returned to the
1698 if (OVERRIDE_CACHE_ATTR(ap
->ah_xfermodes
)) {
1701 i_ddi_cacheattr_to_hatacc(IOMEM_DATA_CACHED
, &order
);
1702 e
= kmem_override_cache_attrs(kaddr
, ap
->ah_len
, order
);
1704 cmn_err(CE_WARN
, "i_ddi_mem_free() failed to "
1705 "override cache attrs, memory leaked\n");
1719 i_ddi_ontrap(ddi_acc_handle_t hp
)
1721 return (DDI_FAILURE
);
1726 i_ddi_notrap(ddi_acc_handle_t hp
)
1736 * Implementation instance override functions
1738 * No override on i86pc
1742 impl_assign_instance(dev_info_t
*dip
)
1744 return ((uint_t
)-1);
1749 impl_keep_instance(dev_info_t
*dip
)
1751 return (DDI_FAILURE
);
1756 impl_free_instance(dev_info_t
*dip
)
1758 return (DDI_FAILURE
);
1763 impl_check_cpu(dev_info_t
*devi
)
1765 return (DDI_SUCCESS
);
1769 * Referenced in kernel/cpr/cpr_driver.c: Power off machine.
1770 * Don't know how to power off i86pc.
1777 * Copy name to property_name, since name
1778 * is in the low address range below kernelbase.
1781 copy_boot_str(const char *boot_str
, char *kern_str
, int len
)
1785 while (i
< len
- 1 && boot_str
[i
] != '\0') {
1786 kern_str
[i
] = boot_str
[i
];
1790 kern_str
[i
] = 0; /* null terminate */
1791 if (boot_str
[i
] != '\0')
1793 "boot property string is truncated to %s", kern_str
);
1797 get_boot_properties(void)
1799 extern char hw_provider
[];
1803 char property_name
[50], property_val
[50];
1804 void *bop_staging_area
;
1806 bop_staging_area
= kmem_zalloc(MMU_PAGESIZE
, KM_NOSLEEP
);
1809 * Import "root" properties from the boot.
1811 * We do this by invoking BOP_NEXTPROP until the list
1812 * is completely copied in.
1815 devi
= ddi_root_node();
1816 for (name
= BOP_NEXTPROP(bootops
, ""); /* get first */
1817 name
; /* NULL => DONE */
1818 name
= BOP_NEXTPROP(bootops
, name
)) { /* get next */
1820 /* copy string to memory above kernelbase */
1821 copy_boot_str(name
, property_name
, 50);
1824 * Skip vga properties. They will be picked up later
1825 * by get_vga_properties.
1827 if (strcmp(property_name
, "display-edif-block") == 0 ||
1828 strcmp(property_name
, "display-edif-id") == 0) {
1832 length
= BOP_GETPROPLEN(bootops
, property_name
);
1835 if (length
> MMU_PAGESIZE
) {
1837 "boot property %s longer than 0x%x, ignored\n",
1838 property_name
, MMU_PAGESIZE
);
1841 BOP_GETPROP(bootops
, property_name
, bop_staging_area
);
1842 flags
= do_bsys_getproptype(bootops
, property_name
);
1845 * special properties:
1847 * goes to kernel data structures.
1848 * bios-boot-device and stdout
1849 * goes to hardware property list so it may show up
1850 * in the prtconf -vp output. This is needed by
1851 * Install/Upgrade. Once we fix install upgrade,
1852 * this can be taken out.
1854 if (strcmp(name
, "si-hw-provider") == 0) {
1855 (void) strncpy(hw_provider
, bop_staging_area
, SYS_NMLN
);
1856 hw_provider
[SYS_NMLN
- 1] = '\0';
1859 if (strcmp(name
, "bios-boot-device") == 0) {
1860 copy_boot_str(bop_staging_area
, property_val
, 50);
1861 (void) ndi_prop_update_string(DDI_DEV_T_NONE
, devi
,
1862 property_name
, property_val
);
1865 if (strcmp(name
, "stdout") == 0) {
1866 (void) ndi_prop_update_int(DDI_DEV_T_NONE
, devi
,
1867 property_name
, *((int *)bop_staging_area
));
1871 /* Boolean property */
1873 (void) e_ddi_prop_create(DDI_DEV_T_NONE
, devi
,
1874 DDI_PROP_CANSLEEP
, property_name
, NULL
, 0);
1878 /* Now anything else based on type. */
1880 case DDI_PROP_TYPE_INT
:
1881 if (length
== sizeof (int)) {
1882 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE
,
1883 devi
, property_name
,
1884 *((int *)bop_staging_area
));
1886 (void) e_ddi_prop_update_int_array(
1887 DDI_DEV_T_NONE
, devi
, property_name
,
1888 bop_staging_area
, length
/ sizeof (int));
1891 case DDI_PROP_TYPE_STRING
:
1892 (void) e_ddi_prop_update_string(DDI_DEV_T_NONE
, devi
,
1893 property_name
, bop_staging_area
);
1895 case DDI_PROP_TYPE_BYTE
:
1896 (void) e_ddi_prop_update_byte_array(DDI_DEV_T_NONE
,
1897 devi
, property_name
, bop_staging_area
, length
);
1899 case DDI_PROP_TYPE_INT64
:
1900 if (length
== sizeof (int64_t)) {
1901 (void) e_ddi_prop_update_int64(DDI_DEV_T_NONE
,
1902 devi
, property_name
,
1903 *((int64_t *)bop_staging_area
));
1905 (void) e_ddi_prop_update_int64_array(
1906 DDI_DEV_T_NONE
, devi
, property_name
,
1908 length
/ sizeof (int64_t));
1912 /* Property type unknown, use old prop interface */
1913 (void) e_ddi_prop_create(DDI_DEV_T_NONE
, devi
,
1914 DDI_PROP_CANSLEEP
, property_name
, bop_staging_area
,
1919 kmem_free(bop_staging_area
, MMU_PAGESIZE
);
1923 get_vga_properties(void)
1929 char property_val
[50];
1930 void *bop_staging_area
;
1934 * There really needs to be a better way for identifying various
1935 * console framebuffers and their related issues. Till then,
1936 * check for this one as a replacement to vgatext.
1938 major
= ddi_name_to_major("ragexl");
1939 if (major
== (major_t
)-1) {
1940 major
= ddi_name_to_major("vgatext");
1941 if (major
== (major_t
)-1)
1944 devi
= devnamesp
[major
].dn_head
;
1948 bop_staging_area
= kmem_zalloc(MMU_PAGESIZE
, KM_SLEEP
);
1951 * Import "vga" properties from the boot.
1953 name
= "display-edif-block";
1954 length
= BOP_GETPROPLEN(bootops
, name
);
1955 if (length
> 0 && length
< MMU_PAGESIZE
) {
1956 BOP_GETPROP(bootops
, name
, bop_staging_area
);
1957 (void) ndi_prop_update_byte_array(DDI_DEV_T_NONE
,
1958 devi
, name
, bop_staging_area
, length
);
1962 * kdmconfig is also looking for display-type and
1963 * video-adapter-type. We default to color and svga.
1965 * Could it be "monochrome", "vga"?
1966 * Nah, you've got to come to the 21st century...
1967 * And you can set monitor type manually in kdmconfig
1968 * if you are really an old junky.
1970 (void) ndi_prop_update_string(DDI_DEV_T_NONE
,
1971 devi
, "display-type", "color");
1972 (void) ndi_prop_update_string(DDI_DEV_T_NONE
,
1973 devi
, "video-adapter-type", "svga");
1975 name
= "display-edif-id";
1976 length
= BOP_GETPROPLEN(bootops
, name
);
1977 if (length
> 0 && length
< MMU_PAGESIZE
) {
1978 BOP_GETPROP(bootops
, name
, bop_staging_area
);
1979 copy_boot_str(bop_staging_area
, property_val
, length
);
1980 (void) ndi_prop_update_string(DDI_DEV_T_NONE
,
1981 devi
, name
, property_val
);
1984 kmem_free(bop_staging_area
, MMU_PAGESIZE
);
1989 * This is temporary, but absolutely necessary. If we are being
1990 * booted with a device tree created by the DevConf project's bootconf
1991 * program, then we have device information nodes that reflect
1992 * reality. At this point in time in the Solaris release schedule, the
1993 * kernel drivers aren't prepared for reality. They still depend on their
1994 * own ad-hoc interpretations of the properties created when their .conf
1995 * files were interpreted. These drivers use an "ignore-hardware-nodes"
1996 * property to prevent them from using the nodes passed up from the bootconf
1999 * Trying to assemble root file system drivers as we are booting from
2000 * devconf will fail if the kernel driver is basing its name_addr's on the
2001 * psuedo-node device info while the bootpath passed up from bootconf is using
2002 * reality-based name_addrs. We help the boot along in this case by
2003 * looking at the pre-bootconf bootpath and determining if we would have
2004 * successfully matched if that had been the bootpath we had chosen.
2006 * Note that we only even perform this extra check if we've booted
2007 * using bootconf's 1275 compliant bootpath, this is the boot device, and
2008 * we're trying to match the name_addr specified in the 1275 bootpath.
2011 #define MAXCOMPONENTLEN 32
2014 x86_old_bootpath_name_addr_match(dev_info_t
*cdip
, char *caddr
, char *naddr
)
2017 * There are multiple criteria to be met before we can even
2018 * consider allowing a name_addr match here.
2020 * 1) We must have been booted such that the bootconf program
2021 * created device tree nodes and properties. This can be
2022 * determined by examining the 'bootpath' property. This
2023 * property will be a non-null string iff bootconf was
2024 * involved in the boot.
2026 * 2) The module that we want to match must be the boot device.
2028 * 3) The instance of the module we are thinking of letting be
2029 * our match must be ignoring hardware nodes.
2031 * 4) The name_addr we want to match must be the name_addr
2032 * specified in the 1275 bootpath.
2034 static char bootdev_module
[MAXCOMPONENTLEN
];
2035 static char bootdev_oldmod
[MAXCOMPONENTLEN
];
2036 static char bootdev_newaddr
[MAXCOMPONENTLEN
];
2037 static char bootdev_oldaddr
[MAXCOMPONENTLEN
];
2038 static int quickexit
;
2044 int rv
= DDI_FAILURE
;
2046 if ((ddi_getlongprop(DDI_DEV_T_ANY
, cdip
, DDI_PROP_DONTPASS
,
2047 "devconf-addr", (caddr_t
)&daddr
, &dlen
) == DDI_PROP_SUCCESS
) &&
2048 (ddi_getprop(DDI_DEV_T_ANY
, cdip
, DDI_PROP_DONTPASS
,
2049 "ignore-hardware-nodes", -1) != -1)) {
2050 if (strcmp(daddr
, caddr
) == 0) {
2051 return (DDI_SUCCESS
);
2058 if (bootdev_module
[0] == '\0') {
2059 char *addrp
, *eoaddrp
;
2060 char *busp
, *modp
, *atp
;
2062 int bp1275len
, bplen
;
2064 bp1275
= bp
= addrp
= eoaddrp
= busp
= modp
= atp
= NULL
;
2066 if (ddi_getlongprop(DDI_DEV_T_ANY
,
2067 ddi_root_node(), 0, "bootpath",
2068 (caddr_t
)&bp1275
, &bp1275len
) != DDI_PROP_SUCCESS
||
2071 * We didn't boot from bootconf so we never need to
2072 * do any special matches.
2076 kmem_free(bp1275
, bp1275len
);
2080 if (ddi_getlongprop(DDI_DEV_T_ANY
,
2081 ddi_root_node(), 0, "boot-path",
2082 (caddr_t
)&bp
, &bplen
) != DDI_PROP_SUCCESS
|| bplen
<= 1) {
2084 * No fallback position for matching. This is
2085 * certainly unexpected, but we'll handle it
2089 kmem_free(bp1275
, bp1275len
);
2091 kmem_free(bp
, bplen
);
2096 * Determine boot device module and 1275 name_addr
2098 * bootpath assumed to be of the form /bus/module@name_addr
2100 if (busp
= strchr(bp1275
, '/')) {
2101 if (modp
= strchr(busp
+ 1, '/')) {
2102 if (atp
= strchr(modp
+ 1, '@')) {
2105 if (eoaddrp
= strchr(addrp
, '/'))
2111 if (modp
&& addrp
) {
2112 (void) strncpy(bootdev_module
, modp
+ 1,
2114 bootdev_module
[MAXCOMPONENTLEN
- 1] = '\0';
2116 (void) strncpy(bootdev_newaddr
, addrp
, MAXCOMPONENTLEN
);
2117 bootdev_newaddr
[MAXCOMPONENTLEN
- 1] = '\0';
2120 kmem_free(bp1275
, bp1275len
);
2121 kmem_free(bp
, bplen
);
2126 * Determine fallback name_addr
2128 * 10/3/96 - Also save fallback module name because it
2129 * might actually be different than the current module
2130 * name. E.G., ISA pnp drivers have new names.
2132 * bootpath assumed to be of the form /bus/module@name_addr
2135 if (busp
= strchr(bp
, '/')) {
2136 if (modp
= strchr(busp
+ 1, '/')) {
2137 if (atp
= strchr(modp
+ 1, '@')) {
2140 if (eoaddrp
= strchr(addrp
, '/'))
2146 if (modp
&& addrp
) {
2147 (void) strncpy(bootdev_oldmod
, modp
+ 1,
2149 bootdev_module
[MAXCOMPONENTLEN
- 1] = '\0';
2151 (void) strncpy(bootdev_oldaddr
, addrp
, MAXCOMPONENTLEN
);
2152 bootdev_oldaddr
[MAXCOMPONENTLEN
- 1] = '\0';
2155 /* Free up the bootpath storage now that we're done with it. */
2156 kmem_free(bp1275
, bp1275len
);
2157 kmem_free(bp
, bplen
);
2159 if (bootdev_oldaddr
[0] == '\0') {
2165 if (((lkupname
= ddi_get_name(cdip
)) != NULL
) &&
2166 (strcmp(bootdev_module
, lkupname
) == 0 ||
2167 strcmp(bootdev_oldmod
, lkupname
) == 0) &&
2168 ((ddi_getprop(DDI_DEV_T_ANY
, cdip
, DDI_PROP_DONTPASS
,
2169 "ignore-hardware-nodes", -1) != -1) ||
2170 ignore_hardware_nodes
) &&
2171 strcmp(bootdev_newaddr
, caddr
) == 0 &&
2172 strcmp(bootdev_oldaddr
, naddr
) == 0) {
2180 * Perform a copy from a memory mapped device (whose devinfo pointer is devi)
2181 * separately mapped at devaddr in the kernel to a kernel buffer at kaddr.
2185 e_ddi_copyfromdev(dev_info_t
*devi
,
2186 off_t off
, const void *devaddr
, void *kaddr
, size_t len
)
2188 bcopy(devaddr
, kaddr
, len
);
2193 * Perform a copy to a memory mapped device (whose devinfo pointer is devi)
2194 * separately mapped at devaddr in the kernel from a kernel buffer at kaddr.
2198 e_ddi_copytodev(dev_info_t
*devi
,
2199 off_t off
, const void *kaddr
, void *devaddr
, size_t len
)
2201 bcopy(kaddr
, devaddr
, len
);
2207 poke_mem(peekpoke_ctlops_t
*in_args
)
2209 int err
= DDI_SUCCESS
;
2212 /* Set up protected environment. */
2213 if (!on_trap(&otd
, OT_DATA_ACCESS
)) {
2214 switch (in_args
->size
) {
2215 case sizeof (uint8_t):
2216 *(uint8_t *)(in_args
->dev_addr
) =
2217 *(uint8_t *)in_args
->host_addr
;
2220 case sizeof (uint16_t):
2221 *(uint16_t *)(in_args
->dev_addr
) =
2222 *(uint16_t *)in_args
->host_addr
;
2225 case sizeof (uint32_t):
2226 *(uint32_t *)(in_args
->dev_addr
) =
2227 *(uint32_t *)in_args
->host_addr
;
2230 case sizeof (uint64_t):
2231 *(uint64_t *)(in_args
->dev_addr
) =
2232 *(uint64_t *)in_args
->host_addr
;
2242 /* Take down protected environment. */
2250 peek_mem(peekpoke_ctlops_t
*in_args
)
2252 int err
= DDI_SUCCESS
;
2255 if (!on_trap(&otd
, OT_DATA_ACCESS
)) {
2256 switch (in_args
->size
) {
2257 case sizeof (uint8_t):
2258 *(uint8_t *)in_args
->host_addr
=
2259 *(uint8_t *)in_args
->dev_addr
;
2262 case sizeof (uint16_t):
2263 *(uint16_t *)in_args
->host_addr
=
2264 *(uint16_t *)in_args
->dev_addr
;
2267 case sizeof (uint32_t):
2268 *(uint32_t *)in_args
->host_addr
=
2269 *(uint32_t *)in_args
->dev_addr
;
2272 case sizeof (uint64_t):
2273 *(uint64_t *)in_args
->host_addr
=
2274 *(uint64_t *)in_args
->dev_addr
;
2290 * This is called only to process peek/poke when the DIP is NULL.
2291 * Assume that this is for memory, as nexi take care of device safe accesses.
2294 peekpoke_mem(ddi_ctl_enum_t cmd
, peekpoke_ctlops_t
*in_args
)
2296 return (cmd
== DDI_CTLOPS_PEEK
? peek_mem(in_args
) : poke_mem(in_args
));
2300 * we've just done a cautious put/get. Check if it was successful by
2301 * calling pci_ereport_post() on all puts and for any gets that return -1
2304 pci_peekpoke_check_fma(dev_info_t
*dip
, void *arg
, ddi_ctl_enum_t ctlop
,
2305 void (*scan
)(dev_info_t
*, ddi_fm_error_t
*))
2307 int rval
= DDI_SUCCESS
;
2308 peekpoke_ctlops_t
*in_args
= (peekpoke_ctlops_t
*)arg
;
2310 ddi_acc_impl_t
*hp
= (ddi_acc_impl_t
*)in_args
->handle
;
2311 ddi_acc_hdl_t
*hdlp
= (ddi_acc_hdl_t
*)in_args
->handle
;
2313 int repcount
= in_args
->repcount
;
2315 if (ctlop
== DDI_CTLOPS_POKE
&&
2316 hdlp
->ah_acc
.devacc_attr_access
!= DDI_CAUTIOUS_ACC
)
2317 return (DDI_SUCCESS
);
2319 if (ctlop
== DDI_CTLOPS_PEEK
&&
2320 hdlp
->ah_acc
.devacc_attr_access
!= DDI_CAUTIOUS_ACC
) {
2321 for (; repcount
; repcount
--) {
2322 switch (in_args
->size
) {
2323 case sizeof (uint8_t):
2324 if (*(uint8_t *)in_args
->host_addr
== 0xff)
2327 case sizeof (uint16_t):
2328 if (*(uint16_t *)in_args
->host_addr
== 0xffff)
2331 case sizeof (uint32_t):
2332 if (*(uint32_t *)in_args
->host_addr
==
2336 case sizeof (uint64_t):
2337 if (*(uint64_t *)in_args
->host_addr
==
2344 return (DDI_SUCCESS
);
2347 * for a cautious put or get or a non-cautious get that returned -1 call
2348 * io framework to see if there really was an error
2350 bzero(&de
, sizeof (ddi_fm_error_t
));
2351 de
.fme_version
= DDI_FME_VERSION
;
2352 de
.fme_ena
= fm_ena_generate(0, FM_ENA_FMT1
);
2353 if (hdlp
->ah_acc
.devacc_attr_access
== DDI_CAUTIOUS_ACC
) {
2354 de
.fme_flag
= DDI_FM_ERR_EXPECTED
;
2355 de
.fme_acc_handle
= in_args
->handle
;
2356 } else if (hdlp
->ah_acc
.devacc_attr_access
== DDI_DEFAULT_ACC
) {
2358 * We only get here with DDI_DEFAULT_ACC for config space gets.
2359 * Non-hardened drivers may be probing the hardware and
2360 * expecting -1 returned. So need to treat errors on
2361 * DDI_DEFAULT_ACC as DDI_FM_ERR_EXPECTED.
2363 de
.fme_flag
= DDI_FM_ERR_EXPECTED
;
2364 de
.fme_acc_handle
= in_args
->handle
;
2367 * Hardened driver doing protected accesses shouldn't
2368 * get errors unless there's a hardware problem. Treat
2369 * as nonfatal if there's an error, but set UNEXPECTED
2370 * so we raise ereports on any errors and potentially
2373 de
.fme_flag
= DDI_FM_ERR_UNEXPECTED
;
2375 (void) scan(dip
, &de
);
2376 if (hdlp
->ah_acc
.devacc_attr_access
!= DDI_DEFAULT_ACC
&&
2377 de
.fme_status
!= DDI_FM_OK
) {
2378 ndi_err_t
*errp
= (ndi_err_t
*)hp
->ahi_err
;
2380 errp
->err_ena
= de
.fme_ena
;
2381 errp
->err_expected
= de
.fme_flag
;
2382 errp
->err_status
= DDI_FM_NONFATAL
;
2388 * pci_peekpoke_check_nofma() is for when an error occurs on a register access
2389 * during pci_ereport_post(). We can't call pci_ereport_post() again or we'd
2390 * recurse, so assume all puts are OK and gets have failed if they return -1
2393 pci_peekpoke_check_nofma(void *arg
, ddi_ctl_enum_t ctlop
)
2395 int rval
= DDI_SUCCESS
;
2396 peekpoke_ctlops_t
*in_args
= (peekpoke_ctlops_t
*)arg
;
2397 ddi_acc_impl_t
*hp
= (ddi_acc_impl_t
*)in_args
->handle
;
2398 ddi_acc_hdl_t
*hdlp
= (ddi_acc_hdl_t
*)in_args
->handle
;
2399 int repcount
= in_args
->repcount
;
2401 if (ctlop
== DDI_CTLOPS_POKE
)
2404 for (; repcount
; repcount
--) {
2405 switch (in_args
->size
) {
2406 case sizeof (uint8_t):
2407 if (*(uint8_t *)in_args
->host_addr
== 0xff)
2410 case sizeof (uint16_t):
2411 if (*(uint16_t *)in_args
->host_addr
== 0xffff)
2414 case sizeof (uint32_t):
2415 if (*(uint32_t *)in_args
->host_addr
== 0xffffffff)
2418 case sizeof (uint64_t):
2419 if (*(uint64_t *)in_args
->host_addr
==
2425 if (hdlp
->ah_acc
.devacc_attr_access
!= DDI_DEFAULT_ACC
&&
2426 rval
== DDI_FAILURE
) {
2427 ndi_err_t
*errp
= (ndi_err_t
*)hp
->ahi_err
;
2428 errp
->err_ena
= fm_ena_generate(0, FM_ENA_FMT1
);
2429 errp
->err_expected
= DDI_FM_ERR_UNEXPECTED
;
2430 errp
->err_status
= DDI_FM_NONFATAL
;
2436 pci_peekpoke_check(dev_info_t
*dip
, dev_info_t
*rdip
,
2437 ddi_ctl_enum_t ctlop
, void *arg
, void *result
,
2438 int (*handler
)(dev_info_t
*, dev_info_t
*, ddi_ctl_enum_t
, void *,
2439 void *), kmutex_t
*err_mutexp
, kmutex_t
*peek_poke_mutexp
,
2440 void (*scan
)(dev_info_t
*, ddi_fm_error_t
*))
2443 peekpoke_ctlops_t
*in_args
= (peekpoke_ctlops_t
*)arg
;
2444 ddi_acc_impl_t
*hp
= (ddi_acc_impl_t
*)in_args
->handle
;
2447 * this function only supports cautious accesses, not peeks/pokes
2448 * which don't have a handle
2451 return (DDI_FAILURE
);
2453 if (hp
->ahi_acc_attr
& DDI_ACCATTR_CONFIG_SPACE
) {
2454 if (!mutex_tryenter(err_mutexp
)) {
2456 * As this may be a recursive call from within
2457 * pci_ereport_post() we can't wait for the mutexes.
2458 * Fortunately we know someone is already calling
2459 * pci_ereport_post() which will handle the error bits
2460 * for us, and as this is a config space access we can
2461 * just do the access and check return value for -1
2462 * using pci_peekpoke_check_nofma().
2464 rval
= handler(dip
, rdip
, ctlop
, arg
, result
);
2465 if (rval
== DDI_SUCCESS
)
2466 rval
= pci_peekpoke_check_nofma(arg
, ctlop
);
2470 * This can't be a recursive call. Drop the err_mutex and get
2471 * both mutexes in the right order. If an error hasn't already
2472 * been detected by the ontrap code, use pci_peekpoke_check_fma
2473 * which will call pci_ereport_post() to check error status.
2475 mutex_exit(err_mutexp
);
2477 mutex_enter(peek_poke_mutexp
);
2478 rval
= handler(dip
, rdip
, ctlop
, arg
, result
);
2479 if (rval
== DDI_SUCCESS
) {
2480 mutex_enter(err_mutexp
);
2481 rval
= pci_peekpoke_check_fma(dip
, arg
, ctlop
, scan
);
2482 mutex_exit(err_mutexp
);
2484 mutex_exit(peek_poke_mutexp
);
2489 impl_setup_ddi(void)
2491 extern void startup_bios_disk(void);
2492 extern int post_fastreboot
;
2493 dev_info_t
*xdip
, *isa_dip
;
2494 rd_existing_t rd_mem_prop
;
2497 ndi_devi_alloc_sleep(ddi_root_node(), "ramdisk",
2498 (pnode_t
)DEVI_SID_NODEID
, &xdip
);
2500 (void) BOP_GETPROP(bootops
,
2501 "ramdisk_start", (void *)&ramdisk_start
);
2502 (void) BOP_GETPROP(bootops
,
2503 "ramdisk_end", (void *)&ramdisk_end
);
2505 rd_mem_prop
.phys
= ramdisk_start
;
2506 rd_mem_prop
.size
= ramdisk_end
- ramdisk_start
+ 1;
2508 (void) ndi_prop_update_byte_array(DDI_DEV_T_NONE
, xdip
,
2509 RD_EXISTING_PROP_NAME
, (uchar_t
*)&rd_mem_prop
,
2510 sizeof (rd_mem_prop
));
2511 err
= ndi_devi_bind_driver(xdip
, 0);
2516 ndi_devi_alloc_sleep(ddi_root_node(), "isa",
2517 (pnode_t
)DEVI_SID_NODEID
, &isa_dip
);
2518 (void) ndi_prop_update_string(DDI_DEV_T_NONE
, isa_dip
,
2519 "device_type", "isa");
2520 (void) ndi_prop_update_string(DDI_DEV_T_NONE
, isa_dip
,
2522 (void) ndi_devi_bind_driver(isa_dip
, 0);
2526 * Read in the properties from the boot.
2528 get_boot_properties();
2530 /* not framebuffer should be enumerated, if present */
2531 get_vga_properties();
2534 * Check for administratively disabled drivers.
2536 check_driver_disable();
2538 if (!post_fastreboot
&& BOP_GETPROPLEN(bootops
, "efi-systab") < 0)
2539 startup_bios_disk();
2540 /* do bus dependent probes. */
2541 impl_bus_initialprobe();
2548 * Usually rootfs.bo_name is initialized by the
2549 * the bootpath property from bootenv.rc, but
2550 * defaults to "/ramdisk:a" otherwise.
2552 return (ddi_pathname_to_dev_t(rootfs
.bo_name
));
2555 static struct bus_probe
{
2556 struct bus_probe
*next
;
2561 impl_bus_add_probe(void (*func
)(int))
2563 struct bus_probe
*probe
;
2564 struct bus_probe
*lastprobe
= NULL
;
2566 probe
= kmem_alloc(sizeof (*probe
), KM_SLEEP
);
2567 probe
->probe
= func
;
2575 lastprobe
= bus_probes
;
2576 while (lastprobe
->next
)
2577 lastprobe
= lastprobe
->next
;
2578 lastprobe
->next
= probe
;
2583 impl_bus_delete_probe(void (*func
)(int))
2585 struct bus_probe
*prev
= NULL
;
2586 struct bus_probe
*probe
= bus_probes
;
2589 if (probe
->probe
== func
)
2592 probe
= probe
->next
;
2599 prev
->next
= probe
->next
;
2601 bus_probes
= probe
->next
;
2603 kmem_free(probe
, sizeof (struct bus_probe
));
2607 * impl_bus_initialprobe
2608 * Modload the prom simulator, then let it probe to verify existence
2609 * and type of PCI support.
2612 impl_bus_initialprobe(void)
2614 struct bus_probe
*probe
;
2616 /* load modules to install bus probes */
2617 if (modload("misc", "pci_autoconfig") < 0) {
2618 panic("failed to load misc/pci_autoconfig");
2621 (void) modload("misc", "acpidev");
2623 if (modload("drv", "isa") < 0)
2624 panic("failed to load drv/isa");
2628 /* run the probe functions */
2630 probe
= probe
->next
;
2636 * Reprogram devices not set up by firmware.
2639 impl_bus_reprobe(void)
2641 struct bus_probe
*probe
;
2645 /* run the probe function */
2647 probe
= probe
->next
;
2653 * The following functions ready a cautious request to go up to the nexus
2654 * driver. It is up to the nexus driver to decide how to process the request.
2655 * It may choose to call i_ddi_do_caut_get/put in this file, or do it
2660 i_ddi_caut_getput_ctlops(ddi_acc_impl_t
*hp
, uint64_t host_addr
,
2661 uint64_t dev_addr
, size_t size
, size_t repcount
, uint_t flags
,
2664 peekpoke_ctlops_t cautacc_ctlops_arg
;
2666 cautacc_ctlops_arg
.size
= size
;
2667 cautacc_ctlops_arg
.dev_addr
= dev_addr
;
2668 cautacc_ctlops_arg
.host_addr
= host_addr
;
2669 cautacc_ctlops_arg
.handle
= (ddi_acc_handle_t
)hp
;
2670 cautacc_ctlops_arg
.repcount
= repcount
;
2671 cautacc_ctlops_arg
.flags
= flags
;
2673 (void) ddi_ctlops(hp
->ahi_common
.ah_dip
, hp
->ahi_common
.ah_dip
, cmd
,
2674 &cautacc_ctlops_arg
, NULL
);
2678 i_ddi_caut_get8(ddi_acc_impl_t
*hp
, uint8_t *addr
)
2681 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2682 sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK
);
2688 i_ddi_caut_get16(ddi_acc_impl_t
*hp
, uint16_t *addr
)
2691 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2692 sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK
);
2698 i_ddi_caut_get32(ddi_acc_impl_t
*hp
, uint32_t *addr
)
2701 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2702 sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK
);
2708 i_ddi_caut_get64(ddi_acc_impl_t
*hp
, uint64_t *addr
)
2711 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2712 sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK
);
2718 i_ddi_caut_put8(ddi_acc_impl_t
*hp
, uint8_t *addr
, uint8_t value
)
2720 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2721 sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE
);
2725 i_ddi_caut_put16(ddi_acc_impl_t
*hp
, uint16_t *addr
, uint16_t value
)
2727 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2728 sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE
);
2732 i_ddi_caut_put32(ddi_acc_impl_t
*hp
, uint32_t *addr
, uint32_t value
)
2734 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2735 sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE
);
2739 i_ddi_caut_put64(ddi_acc_impl_t
*hp
, uint64_t *addr
, uint64_t value
)
2741 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)&value
, (uintptr_t)addr
,
2742 sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE
);
2746 i_ddi_caut_rep_get8(ddi_acc_impl_t
*hp
, uint8_t *host_addr
, uint8_t *dev_addr
,
2747 size_t repcount
, uint_t flags
)
2749 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2750 sizeof (uint8_t), repcount
, flags
, DDI_CTLOPS_PEEK
);
2754 i_ddi_caut_rep_get16(ddi_acc_impl_t
*hp
, uint16_t *host_addr
,
2755 uint16_t *dev_addr
, size_t repcount
, uint_t flags
)
2757 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2758 sizeof (uint16_t), repcount
, flags
, DDI_CTLOPS_PEEK
);
2762 i_ddi_caut_rep_get32(ddi_acc_impl_t
*hp
, uint32_t *host_addr
,
2763 uint32_t *dev_addr
, size_t repcount
, uint_t flags
)
2765 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2766 sizeof (uint32_t), repcount
, flags
, DDI_CTLOPS_PEEK
);
2770 i_ddi_caut_rep_get64(ddi_acc_impl_t
*hp
, uint64_t *host_addr
,
2771 uint64_t *dev_addr
, size_t repcount
, uint_t flags
)
2773 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2774 sizeof (uint64_t), repcount
, flags
, DDI_CTLOPS_PEEK
);
2778 i_ddi_caut_rep_put8(ddi_acc_impl_t
*hp
, uint8_t *host_addr
, uint8_t *dev_addr
,
2779 size_t repcount
, uint_t flags
)
2781 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2782 sizeof (uint8_t), repcount
, flags
, DDI_CTLOPS_POKE
);
2786 i_ddi_caut_rep_put16(ddi_acc_impl_t
*hp
, uint16_t *host_addr
,
2787 uint16_t *dev_addr
, size_t repcount
, uint_t flags
)
2789 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2790 sizeof (uint16_t), repcount
, flags
, DDI_CTLOPS_POKE
);
2794 i_ddi_caut_rep_put32(ddi_acc_impl_t
*hp
, uint32_t *host_addr
,
2795 uint32_t *dev_addr
, size_t repcount
, uint_t flags
)
2797 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2798 sizeof (uint32_t), repcount
, flags
, DDI_CTLOPS_POKE
);
2802 i_ddi_caut_rep_put64(ddi_acc_impl_t
*hp
, uint64_t *host_addr
,
2803 uint64_t *dev_addr
, size_t repcount
, uint_t flags
)
2805 i_ddi_caut_getput_ctlops(hp
, (uintptr_t)host_addr
, (uintptr_t)dev_addr
,
2806 sizeof (uint64_t), repcount
, flags
, DDI_CTLOPS_POKE
);
2810 i_ddi_copybuf_required(ddi_dma_attr_t
*attrp
)
2814 hi_pa
= ((uint64_t)physmax
+ 1ull) << PAGESHIFT
;
2815 if (attrp
->dma_attr_addr_hi
< hi_pa
) {
2823 i_ddi_copybuf_size()
2825 return (dma_max_copybuf_size
);
2830 * returns the maximum DMA size which can be performed in a single DMA
2831 * window taking into account the devices DMA contraints (attrp), the
2832 * maximum copy buffer size (if applicable), and the worse case buffer
2837 i_ddi_dma_max(dev_info_t
*dip
, ddi_dma_attr_t
*attrp
)
2843 * take the min of maxxfer and the the worse case fragementation
2844 * (e.g. every cookie <= 1 page)
2846 maxxfer
= MIN(attrp
->dma_attr_maxxfer
,
2847 ((uint64_t)(attrp
->dma_attr_sgllen
- 1) << PAGESHIFT
));
2850 * If the DMA engine can't reach all off memory, we also need to take
2851 * the max size of the copybuf into consideration.
2853 if (i_ddi_copybuf_required(attrp
)) {
2854 maxxfer
= MIN(i_ddi_copybuf_size(), maxxfer
);
2858 * we only return a 32-bit value. Make sure it's not -1. Round to a
2859 * page so it won't be mistaken for an error value during debug.
2861 if (maxxfer
>= 0xFFFFFFFF) {
2862 maxxfer
= 0xFFFFF000;
2866 * make sure the value we return is a whole multiple of the
2869 if (attrp
->dma_attr_granular
> 1) {
2870 maxxfer
= maxxfer
- (maxxfer
% attrp
->dma_attr_granular
);
2873 return ((uint32_t)maxxfer
);
2878 translate_devid(dev_info_t
*dip
)
2883 i_ddi_paddr_to_pfn(paddr_t paddr
)
2887 pfn
= mmu_btop(paddr
);