2 * Copyright (c) 1991 The Regents of the University of California.
3 * Copyright (c) 1996, by Steve Passe. All rights reserved.
4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved.
7 * This code is derived from software contributed to The DragonFly Project
8 * by Matthew Dillon <dillon@backplane.com>
10 * This code is derived from software contributed to Berkeley by
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
23 * 3. Neither the name of The DragonFly Project nor the names of its
24 * contributors may be used to endorse or promote products derived
25 * from this software without specific, prior written permission.
27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/kernel.h>
44 #include <sys/machintr.h>
45 #include <sys/interrupt.h>
48 #include <sys/thread2.h>
50 #include <machine/smp.h>
51 #include <machine/segments.h>
52 #include <machine/md_var.h>
53 #include <machine/intr_machdep.h>
54 #include <machine/globaldata.h>
55 #include <machine/msi_var.h>
57 #include <machine_base/isa/isa_intr.h>
58 #include <machine_base/icu/icu.h>
59 #include <machine_base/icu/icu_var.h>
60 #include <machine_base/apic/ioapic.h>
61 #include <machine_base/apic/ioapic_abi.h>
62 #include <machine_base/apic/ioapic_ipl.h>
63 #include <machine_base/apic/apicreg.h>
65 #include <dev/acpica/acpi_sci_var.h>
67 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS
80 IDTVEC(ioapic_intr10
),
81 IDTVEC(ioapic_intr11
),
82 IDTVEC(ioapic_intr12
),
83 IDTVEC(ioapic_intr13
),
84 IDTVEC(ioapic_intr14
),
85 IDTVEC(ioapic_intr15
),
86 IDTVEC(ioapic_intr16
),
87 IDTVEC(ioapic_intr17
),
88 IDTVEC(ioapic_intr18
),
89 IDTVEC(ioapic_intr19
),
90 IDTVEC(ioapic_intr20
),
91 IDTVEC(ioapic_intr21
),
92 IDTVEC(ioapic_intr22
),
93 IDTVEC(ioapic_intr23
),
94 IDTVEC(ioapic_intr24
),
95 IDTVEC(ioapic_intr25
),
96 IDTVEC(ioapic_intr26
),
97 IDTVEC(ioapic_intr27
),
98 IDTVEC(ioapic_intr28
),
99 IDTVEC(ioapic_intr29
),
100 IDTVEC(ioapic_intr30
),
101 IDTVEC(ioapic_intr31
),
102 IDTVEC(ioapic_intr32
),
103 IDTVEC(ioapic_intr33
),
104 IDTVEC(ioapic_intr34
),
105 IDTVEC(ioapic_intr35
),
106 IDTVEC(ioapic_intr36
),
107 IDTVEC(ioapic_intr37
),
108 IDTVEC(ioapic_intr38
),
109 IDTVEC(ioapic_intr39
),
110 IDTVEC(ioapic_intr40
),
111 IDTVEC(ioapic_intr41
),
112 IDTVEC(ioapic_intr42
),
113 IDTVEC(ioapic_intr43
),
114 IDTVEC(ioapic_intr44
),
115 IDTVEC(ioapic_intr45
),
116 IDTVEC(ioapic_intr46
),
117 IDTVEC(ioapic_intr47
),
118 IDTVEC(ioapic_intr48
),
119 IDTVEC(ioapic_intr49
),
120 IDTVEC(ioapic_intr50
),
121 IDTVEC(ioapic_intr51
),
122 IDTVEC(ioapic_intr52
),
123 IDTVEC(ioapic_intr53
),
124 IDTVEC(ioapic_intr54
),
125 IDTVEC(ioapic_intr55
),
126 IDTVEC(ioapic_intr56
),
127 IDTVEC(ioapic_intr57
),
128 IDTVEC(ioapic_intr58
),
129 IDTVEC(ioapic_intr59
),
130 IDTVEC(ioapic_intr60
),
131 IDTVEC(ioapic_intr61
),
132 IDTVEC(ioapic_intr62
),
133 IDTVEC(ioapic_intr63
),
134 IDTVEC(ioapic_intr64
),
135 IDTVEC(ioapic_intr65
),
136 IDTVEC(ioapic_intr66
),
137 IDTVEC(ioapic_intr67
),
138 IDTVEC(ioapic_intr68
),
139 IDTVEC(ioapic_intr69
),
140 IDTVEC(ioapic_intr70
),
141 IDTVEC(ioapic_intr71
),
142 IDTVEC(ioapic_intr72
),
143 IDTVEC(ioapic_intr73
),
144 IDTVEC(ioapic_intr74
),
145 IDTVEC(ioapic_intr75
),
146 IDTVEC(ioapic_intr76
),
147 IDTVEC(ioapic_intr77
),
148 IDTVEC(ioapic_intr78
),
149 IDTVEC(ioapic_intr79
),
150 IDTVEC(ioapic_intr80
),
151 IDTVEC(ioapic_intr81
),
152 IDTVEC(ioapic_intr82
),
153 IDTVEC(ioapic_intr83
),
154 IDTVEC(ioapic_intr84
),
155 IDTVEC(ioapic_intr85
),
156 IDTVEC(ioapic_intr86
),
157 IDTVEC(ioapic_intr87
),
158 IDTVEC(ioapic_intr88
),
159 IDTVEC(ioapic_intr89
),
160 IDTVEC(ioapic_intr90
),
161 IDTVEC(ioapic_intr91
),
162 IDTVEC(ioapic_intr92
),
163 IDTVEC(ioapic_intr93
),
164 IDTVEC(ioapic_intr94
),
165 IDTVEC(ioapic_intr95
),
166 IDTVEC(ioapic_intr96
),
167 IDTVEC(ioapic_intr97
),
168 IDTVEC(ioapic_intr98
),
169 IDTVEC(ioapic_intr99
),
170 IDTVEC(ioapic_intr100
),
171 IDTVEC(ioapic_intr101
),
172 IDTVEC(ioapic_intr102
),
173 IDTVEC(ioapic_intr103
),
174 IDTVEC(ioapic_intr104
),
175 IDTVEC(ioapic_intr105
),
176 IDTVEC(ioapic_intr106
),
177 IDTVEC(ioapic_intr107
),
178 IDTVEC(ioapic_intr108
),
179 IDTVEC(ioapic_intr109
),
180 IDTVEC(ioapic_intr110
),
181 IDTVEC(ioapic_intr111
),
182 IDTVEC(ioapic_intr112
),
183 IDTVEC(ioapic_intr113
),
184 IDTVEC(ioapic_intr114
),
185 IDTVEC(ioapic_intr115
),
186 IDTVEC(ioapic_intr116
),
187 IDTVEC(ioapic_intr117
),
188 IDTVEC(ioapic_intr118
),
189 IDTVEC(ioapic_intr119
),
190 IDTVEC(ioapic_intr120
),
191 IDTVEC(ioapic_intr121
),
192 IDTVEC(ioapic_intr122
),
193 IDTVEC(ioapic_intr123
),
194 IDTVEC(ioapic_intr124
),
195 IDTVEC(ioapic_intr125
),
196 IDTVEC(ioapic_intr126
),
197 IDTVEC(ioapic_intr127
),
198 IDTVEC(ioapic_intr128
),
199 IDTVEC(ioapic_intr129
),
200 IDTVEC(ioapic_intr130
),
201 IDTVEC(ioapic_intr131
),
202 IDTVEC(ioapic_intr132
),
203 IDTVEC(ioapic_intr133
),
204 IDTVEC(ioapic_intr134
),
205 IDTVEC(ioapic_intr135
),
206 IDTVEC(ioapic_intr136
),
207 IDTVEC(ioapic_intr137
),
208 IDTVEC(ioapic_intr138
),
209 IDTVEC(ioapic_intr139
),
210 IDTVEC(ioapic_intr140
),
211 IDTVEC(ioapic_intr141
),
212 IDTVEC(ioapic_intr142
),
213 IDTVEC(ioapic_intr143
),
214 IDTVEC(ioapic_intr144
),
215 IDTVEC(ioapic_intr145
),
216 IDTVEC(ioapic_intr146
),
217 IDTVEC(ioapic_intr147
),
218 IDTVEC(ioapic_intr148
),
219 IDTVEC(ioapic_intr149
),
220 IDTVEC(ioapic_intr150
),
221 IDTVEC(ioapic_intr151
),
222 IDTVEC(ioapic_intr152
),
223 IDTVEC(ioapic_intr153
),
224 IDTVEC(ioapic_intr154
),
225 IDTVEC(ioapic_intr155
),
226 IDTVEC(ioapic_intr156
),
227 IDTVEC(ioapic_intr157
),
228 IDTVEC(ioapic_intr158
),
229 IDTVEC(ioapic_intr159
),
230 IDTVEC(ioapic_intr160
),
231 IDTVEC(ioapic_intr161
),
232 IDTVEC(ioapic_intr162
),
233 IDTVEC(ioapic_intr163
),
234 IDTVEC(ioapic_intr164
),
235 IDTVEC(ioapic_intr165
),
236 IDTVEC(ioapic_intr166
),
237 IDTVEC(ioapic_intr167
),
238 IDTVEC(ioapic_intr168
),
239 IDTVEC(ioapic_intr169
),
240 IDTVEC(ioapic_intr170
),
241 IDTVEC(ioapic_intr171
),
242 IDTVEC(ioapic_intr172
),
243 IDTVEC(ioapic_intr173
),
244 IDTVEC(ioapic_intr174
),
245 IDTVEC(ioapic_intr175
),
246 IDTVEC(ioapic_intr176
),
247 IDTVEC(ioapic_intr177
),
248 IDTVEC(ioapic_intr178
),
249 IDTVEC(ioapic_intr179
),
250 IDTVEC(ioapic_intr180
),
251 IDTVEC(ioapic_intr181
),
252 IDTVEC(ioapic_intr182
),
253 IDTVEC(ioapic_intr183
),
254 IDTVEC(ioapic_intr184
),
255 IDTVEC(ioapic_intr185
),
256 IDTVEC(ioapic_intr186
),
257 IDTVEC(ioapic_intr187
),
258 IDTVEC(ioapic_intr188
),
259 IDTVEC(ioapic_intr189
),
260 IDTVEC(ioapic_intr190
),
261 IDTVEC(ioapic_intr191
);
263 static inthand_t
*ioapic_intr
[IOAPIC_HWI_VECTORS
] = {
264 &IDTVEC(ioapic_intr0
),
265 &IDTVEC(ioapic_intr1
),
266 &IDTVEC(ioapic_intr2
),
267 &IDTVEC(ioapic_intr3
),
268 &IDTVEC(ioapic_intr4
),
269 &IDTVEC(ioapic_intr5
),
270 &IDTVEC(ioapic_intr6
),
271 &IDTVEC(ioapic_intr7
),
272 &IDTVEC(ioapic_intr8
),
273 &IDTVEC(ioapic_intr9
),
274 &IDTVEC(ioapic_intr10
),
275 &IDTVEC(ioapic_intr11
),
276 &IDTVEC(ioapic_intr12
),
277 &IDTVEC(ioapic_intr13
),
278 &IDTVEC(ioapic_intr14
),
279 &IDTVEC(ioapic_intr15
),
280 &IDTVEC(ioapic_intr16
),
281 &IDTVEC(ioapic_intr17
),
282 &IDTVEC(ioapic_intr18
),
283 &IDTVEC(ioapic_intr19
),
284 &IDTVEC(ioapic_intr20
),
285 &IDTVEC(ioapic_intr21
),
286 &IDTVEC(ioapic_intr22
),
287 &IDTVEC(ioapic_intr23
),
288 &IDTVEC(ioapic_intr24
),
289 &IDTVEC(ioapic_intr25
),
290 &IDTVEC(ioapic_intr26
),
291 &IDTVEC(ioapic_intr27
),
292 &IDTVEC(ioapic_intr28
),
293 &IDTVEC(ioapic_intr29
),
294 &IDTVEC(ioapic_intr30
),
295 &IDTVEC(ioapic_intr31
),
296 &IDTVEC(ioapic_intr32
),
297 &IDTVEC(ioapic_intr33
),
298 &IDTVEC(ioapic_intr34
),
299 &IDTVEC(ioapic_intr35
),
300 &IDTVEC(ioapic_intr36
),
301 &IDTVEC(ioapic_intr37
),
302 &IDTVEC(ioapic_intr38
),
303 &IDTVEC(ioapic_intr39
),
304 &IDTVEC(ioapic_intr40
),
305 &IDTVEC(ioapic_intr41
),
306 &IDTVEC(ioapic_intr42
),
307 &IDTVEC(ioapic_intr43
),
308 &IDTVEC(ioapic_intr44
),
309 &IDTVEC(ioapic_intr45
),
310 &IDTVEC(ioapic_intr46
),
311 &IDTVEC(ioapic_intr47
),
312 &IDTVEC(ioapic_intr48
),
313 &IDTVEC(ioapic_intr49
),
314 &IDTVEC(ioapic_intr50
),
315 &IDTVEC(ioapic_intr51
),
316 &IDTVEC(ioapic_intr52
),
317 &IDTVEC(ioapic_intr53
),
318 &IDTVEC(ioapic_intr54
),
319 &IDTVEC(ioapic_intr55
),
320 &IDTVEC(ioapic_intr56
),
321 &IDTVEC(ioapic_intr57
),
322 &IDTVEC(ioapic_intr58
),
323 &IDTVEC(ioapic_intr59
),
324 &IDTVEC(ioapic_intr60
),
325 &IDTVEC(ioapic_intr61
),
326 &IDTVEC(ioapic_intr62
),
327 &IDTVEC(ioapic_intr63
),
328 &IDTVEC(ioapic_intr64
),
329 &IDTVEC(ioapic_intr65
),
330 &IDTVEC(ioapic_intr66
),
331 &IDTVEC(ioapic_intr67
),
332 &IDTVEC(ioapic_intr68
),
333 &IDTVEC(ioapic_intr69
),
334 &IDTVEC(ioapic_intr70
),
335 &IDTVEC(ioapic_intr71
),
336 &IDTVEC(ioapic_intr72
),
337 &IDTVEC(ioapic_intr73
),
338 &IDTVEC(ioapic_intr74
),
339 &IDTVEC(ioapic_intr75
),
340 &IDTVEC(ioapic_intr76
),
341 &IDTVEC(ioapic_intr77
),
342 &IDTVEC(ioapic_intr78
),
343 &IDTVEC(ioapic_intr79
),
344 &IDTVEC(ioapic_intr80
),
345 &IDTVEC(ioapic_intr81
),
346 &IDTVEC(ioapic_intr82
),
347 &IDTVEC(ioapic_intr83
),
348 &IDTVEC(ioapic_intr84
),
349 &IDTVEC(ioapic_intr85
),
350 &IDTVEC(ioapic_intr86
),
351 &IDTVEC(ioapic_intr87
),
352 &IDTVEC(ioapic_intr88
),
353 &IDTVEC(ioapic_intr89
),
354 &IDTVEC(ioapic_intr90
),
355 &IDTVEC(ioapic_intr91
),
356 &IDTVEC(ioapic_intr92
),
357 &IDTVEC(ioapic_intr93
),
358 &IDTVEC(ioapic_intr94
),
359 &IDTVEC(ioapic_intr95
),
360 &IDTVEC(ioapic_intr96
),
361 &IDTVEC(ioapic_intr97
),
362 &IDTVEC(ioapic_intr98
),
363 &IDTVEC(ioapic_intr99
),
364 &IDTVEC(ioapic_intr100
),
365 &IDTVEC(ioapic_intr101
),
366 &IDTVEC(ioapic_intr102
),
367 &IDTVEC(ioapic_intr103
),
368 &IDTVEC(ioapic_intr104
),
369 &IDTVEC(ioapic_intr105
),
370 &IDTVEC(ioapic_intr106
),
371 &IDTVEC(ioapic_intr107
),
372 &IDTVEC(ioapic_intr108
),
373 &IDTVEC(ioapic_intr109
),
374 &IDTVEC(ioapic_intr110
),
375 &IDTVEC(ioapic_intr111
),
376 &IDTVEC(ioapic_intr112
),
377 &IDTVEC(ioapic_intr113
),
378 &IDTVEC(ioapic_intr114
),
379 &IDTVEC(ioapic_intr115
),
380 &IDTVEC(ioapic_intr116
),
381 &IDTVEC(ioapic_intr117
),
382 &IDTVEC(ioapic_intr118
),
383 &IDTVEC(ioapic_intr119
),
384 &IDTVEC(ioapic_intr120
),
385 &IDTVEC(ioapic_intr121
),
386 &IDTVEC(ioapic_intr122
),
387 &IDTVEC(ioapic_intr123
),
388 &IDTVEC(ioapic_intr124
),
389 &IDTVEC(ioapic_intr125
),
390 &IDTVEC(ioapic_intr126
),
391 &IDTVEC(ioapic_intr127
),
392 &IDTVEC(ioapic_intr128
),
393 &IDTVEC(ioapic_intr129
),
394 &IDTVEC(ioapic_intr130
),
395 &IDTVEC(ioapic_intr131
),
396 &IDTVEC(ioapic_intr132
),
397 &IDTVEC(ioapic_intr133
),
398 &IDTVEC(ioapic_intr134
),
399 &IDTVEC(ioapic_intr135
),
400 &IDTVEC(ioapic_intr136
),
401 &IDTVEC(ioapic_intr137
),
402 &IDTVEC(ioapic_intr138
),
403 &IDTVEC(ioapic_intr139
),
404 &IDTVEC(ioapic_intr140
),
405 &IDTVEC(ioapic_intr141
),
406 &IDTVEC(ioapic_intr142
),
407 &IDTVEC(ioapic_intr143
),
408 &IDTVEC(ioapic_intr144
),
409 &IDTVEC(ioapic_intr145
),
410 &IDTVEC(ioapic_intr146
),
411 &IDTVEC(ioapic_intr147
),
412 &IDTVEC(ioapic_intr148
),
413 &IDTVEC(ioapic_intr149
),
414 &IDTVEC(ioapic_intr150
),
415 &IDTVEC(ioapic_intr151
),
416 &IDTVEC(ioapic_intr152
),
417 &IDTVEC(ioapic_intr153
),
418 &IDTVEC(ioapic_intr154
),
419 &IDTVEC(ioapic_intr155
),
420 &IDTVEC(ioapic_intr156
),
421 &IDTVEC(ioapic_intr157
),
422 &IDTVEC(ioapic_intr158
),
423 &IDTVEC(ioapic_intr159
),
424 &IDTVEC(ioapic_intr160
),
425 &IDTVEC(ioapic_intr161
),
426 &IDTVEC(ioapic_intr162
),
427 &IDTVEC(ioapic_intr163
),
428 &IDTVEC(ioapic_intr164
),
429 &IDTVEC(ioapic_intr165
),
430 &IDTVEC(ioapic_intr166
),
431 &IDTVEC(ioapic_intr167
),
432 &IDTVEC(ioapic_intr168
),
433 &IDTVEC(ioapic_intr169
),
434 &IDTVEC(ioapic_intr170
),
435 &IDTVEC(ioapic_intr171
),
436 &IDTVEC(ioapic_intr172
),
437 &IDTVEC(ioapic_intr173
),
438 &IDTVEC(ioapic_intr174
),
439 &IDTVEC(ioapic_intr175
),
440 &IDTVEC(ioapic_intr176
),
441 &IDTVEC(ioapic_intr177
),
442 &IDTVEC(ioapic_intr178
),
443 &IDTVEC(ioapic_intr179
),
444 &IDTVEC(ioapic_intr180
),
445 &IDTVEC(ioapic_intr181
),
446 &IDTVEC(ioapic_intr182
),
447 &IDTVEC(ioapic_intr183
),
448 &IDTVEC(ioapic_intr184
),
449 &IDTVEC(ioapic_intr185
),
450 &IDTVEC(ioapic_intr186
),
451 &IDTVEC(ioapic_intr187
),
452 &IDTVEC(ioapic_intr188
),
453 &IDTVEC(ioapic_intr189
),
454 &IDTVEC(ioapic_intr190
),
455 &IDTVEC(ioapic_intr191
)
458 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET)
461 * NOTE: Initialized before VM so cannot use kmalloc() for this array.
463 static struct ioapic_irqmap
{
464 int im_type
; /* IOAPIC_IMT_ */
465 enum intr_trigger im_trig
;
466 enum intr_polarity im_pola
;
469 uint32_t im_flags
; /* IOAPIC_IMF_ */
470 } ioapic_irqmaps
[MAXCPU
][IOAPIC_HWI_VECTORS
];
472 static struct lwkt_token ioapic_irqmap_tok
=
473 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token
);
475 #define IOAPIC_IMT_UNUSED 0
476 #define IOAPIC_IMT_RESERVED 1
477 #define IOAPIC_IMT_LEGACY 2
478 #define IOAPIC_IMT_SYSCALL 3
479 #define IOAPIC_IMT_MSI 4
480 #define IOAPIC_IMT_MSIX 5
482 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \
483 (map)->im_type != IOAPIC_IMT_SYSCALL)
485 #define IOAPIC_IMF_CONF 0x1
487 extern void IOAPIC_INTREN(int);
488 extern void IOAPIC_INTRDIS(int);
490 extern int imcr_present
;
492 static void ioapic_abi_intr_enable(int);
493 static void ioapic_abi_intr_disable(int);
494 static void ioapic_abi_intr_setup(int, int);
495 static void ioapic_abi_intr_teardown(int);
497 static void ioapic_abi_legacy_intr_config(int,
498 enum intr_trigger
, enum intr_polarity
);
499 static int ioapic_abi_legacy_intr_cpuid(int);
500 static int ioapic_abi_legacy_intr_find(int,
501 enum intr_trigger
, enum intr_polarity
);
502 static int ioapic_abi_legacy_intr_find_bygsi(int,
503 enum intr_trigger
, enum intr_polarity
);
505 static int ioapic_abi_msi_alloc(int [], int, int);
506 static void ioapic_abi_msi_release(const int [], int, int);
507 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int);
508 static int ioapic_abi_msix_alloc(int *, int);
509 static void ioapic_abi_msix_release(int, int);
511 static int ioapic_abi_msi_alloc_intern(int, const char *,
513 static void ioapic_abi_msi_release_intern(int, const char *,
514 const int [], int, int);
516 static void ioapic_abi_finalize(void);
517 static void ioapic_abi_cleanup(void);
518 static void ioapic_abi_setdefault(void);
519 static void ioapic_abi_stabilize(void);
520 static void ioapic_abi_initmap(void);
521 static void ioapic_abi_rman_setup(struct rman
*);
523 static int ioapic_abi_gsi_cpuid(int, int);
524 static int ioapic_unused_legacy_irqmap(void);
525 static int ioapic_is_legacy_irqmap_used(int);
527 struct machintr_abi MachIntrABI_IOAPIC
= {
529 .intr_disable
= ioapic_abi_intr_disable
,
530 .intr_enable
= ioapic_abi_intr_enable
,
531 .intr_setup
= ioapic_abi_intr_setup
,
532 .intr_teardown
= ioapic_abi_intr_teardown
,
534 .legacy_intr_config
= ioapic_abi_legacy_intr_config
,
535 .legacy_intr_cpuid
= ioapic_abi_legacy_intr_cpuid
,
536 .legacy_intr_find
= ioapic_abi_legacy_intr_find
,
537 .legacy_intr_find_bygsi
= ioapic_abi_legacy_intr_find_bygsi
,
539 .msi_alloc
= ioapic_abi_msi_alloc
,
540 .msi_release
= ioapic_abi_msi_release
,
541 .msi_map
= ioapic_abi_msi_map
,
542 .msix_alloc
= ioapic_abi_msix_alloc
,
543 .msix_release
= ioapic_abi_msix_release
,
545 .finalize
= ioapic_abi_finalize
,
546 .cleanup
= ioapic_abi_cleanup
,
547 .setdefault
= ioapic_abi_setdefault
,
548 .stabilize
= ioapic_abi_stabilize
,
549 .initmap
= ioapic_abi_initmap
,
550 .rman_setup
= ioapic_abi_rman_setup
553 static int ioapic_abi_extint_irq
= -1;
554 static int ioapic_abi_legacy_irq_max
;
555 static int ioapic_abi_gsi_balance
= 1;
556 static int ioapic_abi_msi_start
; /* NOTE: for testing only */
558 struct ioapic_irqinfo ioapic_irqs
[IOAPIC_HWI_VECTORS
];
561 ioapic_abi_intr_enable(int irq
)
563 const struct ioapic_irqmap
*map
;
565 KASSERT(irq
>= 0 && irq
< IOAPIC_HWI_VECTORS
,
566 ("ioapic enable, invalid irq %d", irq
));
568 map
= &ioapic_irqmaps
[mycpuid
][irq
];
569 KASSERT(IOAPIC_IMT_ISHWI(map
),
570 ("ioapic enable, not hwi irq %d, type %d, cpu%d",
571 irq
, map
->im_type
, mycpuid
));
572 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
579 ioapic_abi_intr_disable(int irq
)
581 const struct ioapic_irqmap
*map
;
583 KASSERT(irq
>= 0 && irq
< IOAPIC_HWI_VECTORS
,
584 ("ioapic disable, invalid irq %d", irq
));
586 map
= &ioapic_irqmaps
[mycpuid
][irq
];
587 KASSERT(IOAPIC_IMT_ISHWI(map
),
588 ("ioapic disable, not hwi irq %d, type %d, cpu%d",
589 irq
, map
->im_type
, mycpuid
));
590 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
597 ioapic_abi_finalize(void)
599 KKASSERT(MachIntrABI
.type
== MACHINTR_IOAPIC
);
600 KKASSERT(ioapic_enable
);
603 * If an IMCR is present, program bit 0 to disconnect the 8259
607 outb(0x22, 0x70); /* select IMCR */
608 outb(0x23, 0x01); /* disconnect 8259 */
613 * This routine is called after physical interrupts are enabled but before
614 * the critical section is released. We need to clean out any interrupts
615 * that had already been posted to the cpu.
618 ioapic_abi_cleanup(void)
620 bzero(mdcpu
->gd_ipending
, sizeof(mdcpu
->gd_ipending
));
623 /* Must never be called */
625 ioapic_abi_stabilize(void)
627 panic("ioapic_stabilize is called");
631 ioapic_abi_intr_setup(int intr
, int flags
)
633 const struct ioapic_irqmap
*map
;
638 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
639 ("ioapic setup, invalid irq %d", intr
));
641 map
= &ioapic_irqmaps
[mycpuid
][intr
];
642 KASSERT(IOAPIC_IMT_ISHWI(map
),
643 ("ioapic setup, not hwi irq %d, type %d, cpu%d",
644 intr
, map
->im_type
, mycpuid
));
645 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
648 KASSERT(ioapic_irqs
[intr
].io_addr
!= NULL
,
649 ("ioapic setup, no GSI information, irq %d", intr
));
654 vector
= IDT_OFFSET
+ intr
;
657 * Now reprogram the vector in the IO APIC. In order to avoid
658 * losing an EOI for a level interrupt, which is vector based,
659 * make sure that the IO APIC is programmed for edge-triggering
660 * first, then reprogrammed with the new vector. This should
665 select
= ioapic_irqs
[intr
].io_idx
;
666 value
= ioapic_read(ioapic_irqs
[intr
].io_addr
, select
);
667 value
|= IOART_INTMSET
;
669 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
670 (value
& ~APIC_TRIGMOD_MASK
));
671 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
672 (value
& ~IOART_INTVEC
) | vector
);
682 ioapic_abi_intr_teardown(int intr
)
684 const struct ioapic_irqmap
*map
;
689 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
690 ("ioapic teardown, invalid irq %d", intr
));
692 map
= &ioapic_irqmaps
[mycpuid
][intr
];
693 KASSERT(IOAPIC_IMT_ISHWI(map
),
694 ("ioapic teardown, not hwi irq %d, type %d, cpu%d",
695 intr
, map
->im_type
, mycpuid
));
696 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
699 KASSERT(ioapic_irqs
[intr
].io_addr
!= NULL
,
700 ("ioapic teardown, no GSI information, irq %d", intr
));
706 * Teardown an interrupt vector. The vector should already be
707 * installed in the cpu's IDT, but make sure.
709 IOAPIC_INTRDIS(intr
);
711 vector
= IDT_OFFSET
+ intr
;
714 * In order to avoid losing an EOI for a level interrupt, which
715 * is vector based, make sure that the IO APIC is programmed for
716 * edge-triggering first, then reprogrammed with the new vector.
717 * This should clear the IRR bit.
721 select
= ioapic_irqs
[intr
].io_idx
;
722 value
= ioapic_read(ioapic_irqs
[intr
].io_addr
, select
);
724 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
725 (value
& ~APIC_TRIGMOD_MASK
));
726 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
727 (value
& ~IOART_INTVEC
) | vector
);
735 ioapic_abi_setdefault(void)
739 for (intr
= 0; intr
< IOAPIC_HWI_VECTORS
; ++intr
) {
740 if (intr
== IOAPIC_HWI_SYSCALL
)
742 setidt_global(IDT_OFFSET
+ intr
, ioapic_intr
[intr
],
743 SDT_SYSIGT
, SEL_KPL
, 0);
748 ioapic_abi_initmap(void)
752 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance
);
754 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start
);
755 ioapic_abi_msi_start
&= ~0x1f; /* MUST be 32 aligned */
758 * NOTE: ncpus is not ready yet
760 for (cpu
= 0; cpu
< MAXCPU
; ++cpu
) {
763 for (i
= 0; i
< IOAPIC_HWI_VECTORS
; ++i
) {
764 ioapic_irqmaps
[cpu
][i
].im_gsi
= -1;
765 ioapic_irqmaps
[cpu
][i
].im_msi_base
= -1;
767 ioapic_irqmaps
[cpu
][IOAPIC_HWI_SYSCALL
].im_type
=
773 * Only one CPU can have the legacy IRQ map.
776 ioapic_is_legacy_irqmap_used(int irq
)
780 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
781 if (ioapic_irqmaps
[cpu
][irq
].im_type
!= IOAPIC_IMT_UNUSED
)
788 ioapic_unused_legacy_irqmap(void)
792 for (i
= ISA_IRQ_CNT
; i
< IOAPIC_HWI_VECTORS
; ++i
) {
793 if (i
== acpi_sci_irqno())
795 if (!ioapic_is_legacy_irqmap_used(i
))
802 ioapic_set_legacy_irqmap(int irq
, int gsi
, enum intr_trigger trig
,
803 enum intr_polarity pola
)
805 struct ioapic_irqinfo
*info
;
806 struct ioapic_irqmap
*map
;
810 KKASSERT(trig
== INTR_TRIGGER_EDGE
|| trig
== INTR_TRIGGER_LEVEL
);
811 KKASSERT(pola
== INTR_POLARITY_HIGH
|| pola
== INTR_POLARITY_LOW
);
814 if (irq
>= IOAPIC_HWI_VECTORS
) {
816 * Some BIOSes seem to assume that all 256 IDT vectors
817 * could be used, while we limit the available IDT
818 * vectors to 192; find an unused IRQ for this GSI.
820 irq
= ioapic_unused_legacy_irqmap();
822 kprintf("failed to find unused irq for gsi %d, "
827 KKASSERT(irq
< IOAPIC_HWI_VECTORS
);
829 if (ioapic_is_legacy_irqmap_used(irq
)) {
831 * There are so many IOAPICs, that 1:1 mapping
832 * of GSI and IRQ hits SYSCALL entry.
834 irq
= ioapic_unused_legacy_irqmap();
836 kprintf("failed to find unused irq for gsi %d, "
840 KKASSERT(irq
< IOAPIC_HWI_VECTORS
);
843 cpuid
= ioapic_abi_gsi_cpuid(irq
, gsi
);
844 map
= &ioapic_irqmaps
[cpuid
][irq
];
846 if (irq
> ioapic_abi_legacy_irq_max
)
847 ioapic_abi_legacy_irq_max
= irq
;
849 KKASSERT(map
->im_type
== IOAPIC_IMT_UNUSED
);
850 map
->im_type
= IOAPIC_IMT_LEGACY
;
857 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n",
859 intr_str_trigger(map
->im_trig
),
860 intr_str_polarity(map
->im_pola
));
863 pin
= ioapic_gsi_pin(map
->im_gsi
);
864 ioaddr
= ioapic_gsi_ioaddr(map
->im_gsi
);
866 info
= &ioapic_irqs
[irq
];
870 info
->io_addr
= ioaddr
;
871 info
->io_idx
= IOAPIC_REDTBL
+ (2 * pin
);
872 info
->io_flags
= IOAPIC_IRQI_FLAG_MASKED
;
873 if (map
->im_trig
== INTR_TRIGGER_LEVEL
)
874 info
->io_flags
|= IOAPIC_IRQI_FLAG_LEVEL
;
876 ioapic_pin_setup(ioaddr
, pin
, IDT_OFFSET
+ irq
,
877 map
->im_trig
, map
->im_pola
, cpuid
);
883 ioapic_fixup_legacy_irqmaps(void)
887 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
890 for (i
= 0; i
< ISA_IRQ_CNT
; ++i
) {
891 struct ioapic_irqmap
*map
= &ioapic_irqmaps
[cpu
][i
];
893 if (map
->im_type
== IOAPIC_IMT_UNUSED
) {
894 map
->im_type
= IOAPIC_IMT_RESERVED
;
897 "cpu%d irq %d reserved\n", cpu
, i
);
903 ioapic_abi_legacy_irq_max
+= 1;
905 kprintf("IOAPIC: legacy irq max %d\n",
906 ioapic_abi_legacy_irq_max
);
911 ioapic_abi_legacy_intr_find_bygsi(int gsi
, enum intr_trigger trig
,
912 enum intr_polarity pola
)
917 if (trig
== INTR_TRIGGER_CONFORM
) {
918 KKASSERT(pola
== INTR_POLARITY_CONFORM
);
920 KKASSERT(trig
== INTR_TRIGGER_EDGE
||
921 trig
== INTR_TRIGGER_LEVEL
);
922 KKASSERT(pola
== INTR_POLARITY_HIGH
||
923 pola
== INTR_POLARITY_LOW
);
927 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
930 for (irq
= 0; irq
< ioapic_abi_legacy_irq_max
; ++irq
) {
931 const struct ioapic_irqmap
*map
=
932 &ioapic_irqmaps
[cpu
][irq
];
934 if (map
->im_gsi
== gsi
) {
935 KKASSERT(map
->im_type
== IOAPIC_IMT_LEGACY
);
937 if ((map
->im_flags
& IOAPIC_IMF_CONF
) &&
938 trig
!= INTR_TRIGGER_CONFORM
&&
939 pola
!= INTR_POLARITY_CONFORM
) {
940 if (map
->im_trig
!= trig
||
941 map
->im_pola
!= pola
)
952 ioapic_abi_legacy_intr_find(int irq
, enum intr_trigger trig
,
953 enum intr_polarity pola
)
958 if (trig
== INTR_TRIGGER_CONFORM
) {
959 KKASSERT(pola
== INTR_POLARITY_CONFORM
);
961 KKASSERT(trig
== INTR_TRIGGER_EDGE
||
962 trig
== INTR_TRIGGER_LEVEL
);
963 KKASSERT(pola
== INTR_POLARITY_HIGH
||
964 pola
== INTR_POLARITY_LOW
);
968 if (irq
< 0 || irq
>= ioapic_abi_legacy_irq_max
)
971 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
972 const struct ioapic_irqmap
*map
= &ioapic_irqmaps
[cpu
][irq
];
974 if (map
->im_type
== IOAPIC_IMT_LEGACY
) {
975 if ((map
->im_flags
& IOAPIC_IMF_CONF
) &&
976 trig
!= INTR_TRIGGER_CONFORM
&&
977 pola
!= INTR_POLARITY_CONFORM
) {
978 if (map
->im_trig
!= trig
||
979 map
->im_pola
!= pola
)
989 ioapic_abi_legacy_intr_config(int irq
, enum intr_trigger trig
,
990 enum intr_polarity pola
)
992 struct ioapic_irqinfo
*info
;
993 struct ioapic_irqmap
*map
= NULL
;
997 KKASSERT(trig
== INTR_TRIGGER_EDGE
|| trig
== INTR_TRIGGER_LEVEL
);
998 KKASSERT(pola
== INTR_POLARITY_HIGH
|| pola
== INTR_POLARITY_LOW
);
1000 KKASSERT(irq
>= 0 && irq
< ioapic_abi_legacy_irq_max
);
1001 for (cpuid
= 0; cpuid
< ncpus
; ++cpuid
) {
1002 map
= &ioapic_irqmaps
[cpuid
][irq
];
1003 if (map
->im_type
== IOAPIC_IMT_LEGACY
)
1006 KKASSERT(cpuid
< ncpus
);
1009 if (map
->im_flags
& IOAPIC_IMF_CONF
) {
1010 if (trig
!= map
->im_trig
) {
1011 panic("ioapic_intr_config: trig %s -> %s",
1012 intr_str_trigger(map
->im_trig
),
1013 intr_str_trigger(trig
));
1015 if (pola
!= map
->im_pola
) {
1016 panic("ioapic_intr_config: pola %s -> %s",
1017 intr_str_polarity(map
->im_pola
),
1018 intr_str_polarity(pola
));
1023 map
->im_flags
|= IOAPIC_IMF_CONF
;
1025 if (trig
== map
->im_trig
&& pola
== map
->im_pola
)
1029 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n",
1031 intr_str_trigger(map
->im_trig
),
1032 intr_str_polarity(map
->im_pola
),
1033 intr_str_trigger(trig
),
1034 intr_str_polarity(pola
));
1036 map
->im_trig
= trig
;
1037 map
->im_pola
= pola
;
1039 pin
= ioapic_gsi_pin(map
->im_gsi
);
1040 ioaddr
= ioapic_gsi_ioaddr(map
->im_gsi
);
1042 info
= &ioapic_irqs
[irq
];
1046 info
->io_flags
&= ~IOAPIC_IRQI_FLAG_LEVEL
;
1047 if (map
->im_trig
== INTR_TRIGGER_LEVEL
)
1048 info
->io_flags
|= IOAPIC_IRQI_FLAG_LEVEL
;
1050 ioapic_pin_setup(ioaddr
, pin
, IDT_OFFSET
+ irq
,
1051 map
->im_trig
, map
->im_pola
, cpuid
);
1057 ioapic_conf_legacy_extint(int irq
)
1059 struct ioapic_irqinfo
*info
;
1060 struct ioapic_irqmap
*map
;
1062 int pin
, error
, vec
;
1064 /* XXX only irq0 is allowed */
1067 vec
= IDT_OFFSET
+ irq
;
1069 if (ioapic_abi_extint_irq
== irq
)
1071 else if (ioapic_abi_extint_irq
>= 0)
1074 error
= icu_ioapic_extint(irq
, vec
);
1078 /* ExtINT is always targeted to cpu0 */
1079 map
= &ioapic_irqmaps
[0][irq
];
1081 KKASSERT(map
->im_type
== IOAPIC_IMT_RESERVED
||
1082 map
->im_type
== IOAPIC_IMT_LEGACY
);
1083 if (map
->im_type
== IOAPIC_IMT_LEGACY
) {
1084 if (map
->im_flags
& IOAPIC_IMF_CONF
)
1087 ioapic_abi_extint_irq
= irq
;
1089 map
->im_type
= IOAPIC_IMT_LEGACY
;
1090 map
->im_trig
= INTR_TRIGGER_EDGE
;
1091 map
->im_pola
= INTR_POLARITY_HIGH
;
1092 map
->im_flags
= IOAPIC_IMF_CONF
;
1094 map
->im_gsi
= ioapic_extpin_gsi();
1095 KKASSERT(map
->im_gsi
>= 0);
1098 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n",
1100 intr_str_trigger(map
->im_trig
),
1101 intr_str_polarity(map
->im_pola
));
1104 pin
= ioapic_gsi_pin(map
->im_gsi
);
1105 ioaddr
= ioapic_gsi_ioaddr(map
->im_gsi
);
1107 info
= &ioapic_irqs
[irq
];
1111 info
->io_addr
= ioaddr
;
1112 info
->io_idx
= IOAPIC_REDTBL
+ (2 * pin
);
1113 info
->io_flags
= IOAPIC_IRQI_FLAG_MASKED
;
1115 ioapic_extpin_setup(ioaddr
, pin
, vec
);
1123 ioapic_abi_legacy_intr_cpuid(int irq
)
1125 const struct ioapic_irqmap
*map
= NULL
;
1128 KKASSERT(irq
>= 0 && irq
< ioapic_abi_legacy_irq_max
);
1130 for (cpuid
= 0; cpuid
< ncpus
; ++cpuid
) {
1131 map
= &ioapic_irqmaps
[cpuid
][irq
];
1132 if (map
->im_type
== IOAPIC_IMT_LEGACY
)
1136 /* XXX some drivers tries to peek at reserved IRQs */
1137 for (cpuid
= 0; cpuid
< ncpus
; ++cpuid
) {
1138 map
= &ioapic_irqmaps
[cpuid
][irq
];
1139 KKASSERT(map
->im_type
== IOAPIC_IMT_RESERVED
);
1145 ioapic_abi_gsi_cpuid(int irq
, int gsi
)
1152 if (irq
== 0 || gsi
== 0) {
1155 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n",
1161 if (irq
>= 0 && irq
== acpi_sci_irqno()) {
1163 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n",
1169 ksnprintf(envpath
, sizeof(envpath
), "hw.ioapic.gsi.%d.cpu", gsi
);
1170 kgetenv_int(envpath
, &cpuid
);
1173 if (!ioapic_abi_gsi_balance
) {
1174 if (irq
>= 0 && bootverbose
) {
1175 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 "
1176 "(fixed)\n", irq
, gsi
);
1181 cpuid
= gsi
% ncpus
;
1182 if (irq
>= 0 && bootverbose
) {
1183 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n",
1186 } else if (cpuid
>= ncpus
) {
1188 if (irq
>= 0 && bootverbose
) {
1189 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n",
1193 if (irq
>= 0 && bootverbose
) {
1194 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n",
1202 ioapic_abi_rman_setup(struct rman
*rm
)
1206 KASSERT(rm
->rm_cpuid
>= 0 && rm
->rm_cpuid
< MAXCPU
,
1207 ("invalid rman cpuid %d", rm
->rm_cpuid
));
1210 for (i
= 0; i
< IOAPIC_HWI_VECTORS
; ++i
) {
1211 const struct ioapic_irqmap
*map
=
1212 &ioapic_irqmaps
[rm
->rm_cpuid
][i
];
1215 if (IOAPIC_IMT_ISHWI(map
))
1218 if (IOAPIC_IMT_ISHWI(map
)) {
1223 kprintf("IOAPIC: rman cpu%d %d - %d\n",
1224 rm
->rm_cpuid
, start
, end
);
1226 if (rman_manage_region(rm
, start
, end
)) {
1227 panic("rman_manage_region"
1228 "(cpu%d %d - %d)", rm
->rm_cpuid
,
1238 kprintf("IOAPIC: rman cpu%d %d - %d\n",
1239 rm
->rm_cpuid
, start
, end
);
1241 if (rman_manage_region(rm
, start
, end
)) {
1242 panic("rman_manage_region(cpu%d %d - %d)",
1243 rm
->rm_cpuid
, start
, end
);
1249 ioapic_abi_msi_alloc_intern(int type
, const char *desc
,
1250 int intrs
[], int count
, int cpuid
)
1254 KASSERT(cpuid
>= 0 && cpuid
< ncpus
,
1255 ("invalid cpuid %d", cpuid
));
1257 KASSERT(count
> 0 && count
<= 32, ("invalid count %d", count
));
1258 KASSERT((count
& (count
- 1)) == 0,
1259 ("count %d is not power of 2", count
));
1261 lwkt_gettoken(&ioapic_irqmap_tok
);
1265 * Since IDT_OFFSET is 32, which is the maximum valid 'count',
1266 * we do not need to find out the first properly aligned
1271 for (i
= ioapic_abi_msi_start
; i
< IOAPIC_HWI_VECTORS
; i
+= count
) {
1274 if (ioapic_irqmaps
[cpuid
][i
].im_type
!= IOAPIC_IMT_UNUSED
)
1277 for (j
= 1; j
< count
; ++j
) {
1278 if (ioapic_irqmaps
[cpuid
][i
+ j
].im_type
!=
1285 for (j
= 0; j
< count
; ++j
) {
1286 struct ioapic_irqmap
*map
;
1289 map
= &ioapic_irqmaps
[cpuid
][intr
];
1290 KASSERT(map
->im_msi_base
< 0,
1291 ("intr %d, stale %s-base %d",
1292 intr
, desc
, map
->im_msi_base
));
1294 map
->im_type
= type
;
1295 map
->im_msi_base
= i
;
1298 msi_setup(intr
, cpuid
);
1301 kprintf("alloc %s intr %d on cpu%d\n",
1309 lwkt_reltoken(&ioapic_irqmap_tok
);
1315 ioapic_abi_msi_release_intern(int type
, const char *desc
,
1316 const int intrs
[], int count
, int cpuid
)
1318 int i
, msi_base
= -1, intr_next
= -1, mask
;
1320 KASSERT(cpuid
>= 0 && cpuid
< ncpus
,
1321 ("invalid cpuid %d", cpuid
));
1323 KASSERT(count
> 0 && count
<= 32, ("invalid count %d", count
));
1326 KASSERT((count
& mask
) == 0, ("count %d is not power of 2", count
));
1328 lwkt_gettoken(&ioapic_irqmap_tok
);
1330 for (i
= 0; i
< count
; ++i
) {
1331 struct ioapic_irqmap
*map
;
1332 int intr
= intrs
[i
];
1334 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
1335 ("invalid intr %d", intr
));
1337 map
= &ioapic_irqmaps
[cpuid
][intr
];
1338 KASSERT(map
->im_type
== type
,
1339 ("trying to release non-%s intr %d, type %d", desc
,
1340 intr
, map
->im_type
));
1341 KASSERT(map
->im_msi_base
>= 0 && map
->im_msi_base
<= intr
,
1342 ("intr %d, invalid %s-base %d", intr
, desc
,
1344 KASSERT((map
->im_msi_base
& mask
) == 0,
1345 ("intr %d, %s-base %d is not properly aligned %d",
1346 intr
, desc
, map
->im_msi_base
, count
));
1349 msi_base
= map
->im_msi_base
;
1351 KASSERT(map
->im_msi_base
== msi_base
,
1352 ("intr %d, inconsistent %s-base, "
1354 intr
, desc
, msi_base
, map
->im_msi_base
));
1357 if (intr_next
< intr
)
1360 map
->im_type
= IOAPIC_IMT_UNUSED
;
1361 map
->im_msi_base
= -1;
1364 kprintf("release %s intr %d on cpu%d\n",
1369 KKASSERT(intr_next
> 0);
1370 KKASSERT(msi_base
>= 0);
1373 if (intr_next
< IOAPIC_HWI_VECTORS
) {
1374 const struct ioapic_irqmap
*map
=
1375 &ioapic_irqmaps
[cpuid
][intr_next
];
1377 if (map
->im_type
== type
) {
1378 KASSERT(map
->im_msi_base
!= msi_base
,
1379 ("more than %d %s was allocated", count
, desc
));
1383 lwkt_reltoken(&ioapic_irqmap_tok
);
1387 ioapic_abi_msi_alloc(int intrs
[], int count
, int cpuid
)
1389 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI
, "MSI",
1390 intrs
, count
, cpuid
);
1394 ioapic_abi_msi_release(const int intrs
[], int count
, int cpuid
)
1396 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI
, "MSI",
1397 intrs
, count
, cpuid
);
1401 ioapic_abi_msix_alloc(int *intr
, int cpuid
)
1403 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX
, "MSI-X",
1408 ioapic_abi_msix_release(int intr
, int cpuid
)
1410 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX
, "MSI-X",
1415 ioapic_abi_msi_map(int intr
, uint64_t *addr
, uint32_t *data
, int cpuid
)
1417 const struct ioapic_irqmap
*map
;
1419 KASSERT(cpuid
>= 0 && cpuid
< ncpus
,
1420 ("invalid cpuid %d", cpuid
));
1422 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
1423 ("invalid intr %d", intr
));
1425 lwkt_gettoken(&ioapic_irqmap_tok
);
1427 map
= &ioapic_irqmaps
[cpuid
][intr
];
1428 KASSERT(map
->im_type
== IOAPIC_IMT_MSI
||
1429 map
->im_type
== IOAPIC_IMT_MSIX
,
1430 ("trying to map non-MSI/MSI-X intr %d, type %d", intr
, map
->im_type
));
1431 KASSERT(map
->im_msi_base
>= 0 && map
->im_msi_base
<= intr
,
1432 ("intr %d, invalid %s-base %d", intr
,
1433 map
->im_type
== IOAPIC_IMT_MSI
? "MSI" : "MSI-X",
1436 msi_map(map
->im_msi_base
, addr
, data
, cpuid
);
1439 kprintf("map %s intr %d on cpu%d\n",
1440 map
->im_type
== IOAPIC_IMT_MSI
? "MSI" : "MSI-X",
1444 lwkt_reltoken(&ioapic_irqmap_tok
);