2 * Copyright (c) 1991 The Regents of the University of California.
3 * Copyright (c) 1996, by Steve Passe. All rights reserved.
4 * Copyright (c) 2005,2008 The DragonFly Project. All rights reserved.
7 * This code is derived from software contributed to The DragonFly Project
8 * by Matthew Dillon <dillon@backplane.com>
10 * This code is derived from software contributed to Berkeley by
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
23 * 3. Neither the name of The DragonFly Project nor the names of its
24 * contributors may be used to endorse or promote products derived
25 * from this software without specific, prior written permission.
27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
30 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
31 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
32 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
34 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
35 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
37 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/kernel.h>
44 #include <sys/machintr.h>
45 #include <sys/interrupt.h>
49 #include <machine/smp.h>
50 #include <machine/segments.h>
51 #include <machine/md_var.h>
52 #include <machine/intr_machdep.h>
53 #include <machine/globaldata.h>
54 #include <machine/msi_var.h>
56 #include <machine_base/isa/isa_intr.h>
57 #include <machine_base/icu/icu.h>
58 #include <machine_base/icu/icu_var.h>
59 #include <machine_base/apic/ioapic.h>
60 #include <machine_base/apic/ioapic_abi.h>
61 #include <machine_base/apic/ioapic_ipl.h>
62 #include <machine_base/apic/apicreg.h>
64 #include <dev/acpica/acpi_sci_var.h>
66 #define IOAPIC_HWI_VECTORS IDT_HWI_VECTORS
79 IDTVEC(ioapic_intr10
),
80 IDTVEC(ioapic_intr11
),
81 IDTVEC(ioapic_intr12
),
82 IDTVEC(ioapic_intr13
),
83 IDTVEC(ioapic_intr14
),
84 IDTVEC(ioapic_intr15
),
85 IDTVEC(ioapic_intr16
),
86 IDTVEC(ioapic_intr17
),
87 IDTVEC(ioapic_intr18
),
88 IDTVEC(ioapic_intr19
),
89 IDTVEC(ioapic_intr20
),
90 IDTVEC(ioapic_intr21
),
91 IDTVEC(ioapic_intr22
),
92 IDTVEC(ioapic_intr23
),
93 IDTVEC(ioapic_intr24
),
94 IDTVEC(ioapic_intr25
),
95 IDTVEC(ioapic_intr26
),
96 IDTVEC(ioapic_intr27
),
97 IDTVEC(ioapic_intr28
),
98 IDTVEC(ioapic_intr29
),
99 IDTVEC(ioapic_intr30
),
100 IDTVEC(ioapic_intr31
),
101 IDTVEC(ioapic_intr32
),
102 IDTVEC(ioapic_intr33
),
103 IDTVEC(ioapic_intr34
),
104 IDTVEC(ioapic_intr35
),
105 IDTVEC(ioapic_intr36
),
106 IDTVEC(ioapic_intr37
),
107 IDTVEC(ioapic_intr38
),
108 IDTVEC(ioapic_intr39
),
109 IDTVEC(ioapic_intr40
),
110 IDTVEC(ioapic_intr41
),
111 IDTVEC(ioapic_intr42
),
112 IDTVEC(ioapic_intr43
),
113 IDTVEC(ioapic_intr44
),
114 IDTVEC(ioapic_intr45
),
115 IDTVEC(ioapic_intr46
),
116 IDTVEC(ioapic_intr47
),
117 IDTVEC(ioapic_intr48
),
118 IDTVEC(ioapic_intr49
),
119 IDTVEC(ioapic_intr50
),
120 IDTVEC(ioapic_intr51
),
121 IDTVEC(ioapic_intr52
),
122 IDTVEC(ioapic_intr53
),
123 IDTVEC(ioapic_intr54
),
124 IDTVEC(ioapic_intr55
),
125 IDTVEC(ioapic_intr56
),
126 IDTVEC(ioapic_intr57
),
127 IDTVEC(ioapic_intr58
),
128 IDTVEC(ioapic_intr59
),
129 IDTVEC(ioapic_intr60
),
130 IDTVEC(ioapic_intr61
),
131 IDTVEC(ioapic_intr62
),
132 IDTVEC(ioapic_intr63
),
133 IDTVEC(ioapic_intr64
),
134 IDTVEC(ioapic_intr65
),
135 IDTVEC(ioapic_intr66
),
136 IDTVEC(ioapic_intr67
),
137 IDTVEC(ioapic_intr68
),
138 IDTVEC(ioapic_intr69
),
139 IDTVEC(ioapic_intr70
),
140 IDTVEC(ioapic_intr71
),
141 IDTVEC(ioapic_intr72
),
142 IDTVEC(ioapic_intr73
),
143 IDTVEC(ioapic_intr74
),
144 IDTVEC(ioapic_intr75
),
145 IDTVEC(ioapic_intr76
),
146 IDTVEC(ioapic_intr77
),
147 IDTVEC(ioapic_intr78
),
148 IDTVEC(ioapic_intr79
),
149 IDTVEC(ioapic_intr80
),
150 IDTVEC(ioapic_intr81
),
151 IDTVEC(ioapic_intr82
),
152 IDTVEC(ioapic_intr83
),
153 IDTVEC(ioapic_intr84
),
154 IDTVEC(ioapic_intr85
),
155 IDTVEC(ioapic_intr86
),
156 IDTVEC(ioapic_intr87
),
157 IDTVEC(ioapic_intr88
),
158 IDTVEC(ioapic_intr89
),
159 IDTVEC(ioapic_intr90
),
160 IDTVEC(ioapic_intr91
),
161 IDTVEC(ioapic_intr92
),
162 IDTVEC(ioapic_intr93
),
163 IDTVEC(ioapic_intr94
),
164 IDTVEC(ioapic_intr95
),
165 IDTVEC(ioapic_intr96
),
166 IDTVEC(ioapic_intr97
),
167 IDTVEC(ioapic_intr98
),
168 IDTVEC(ioapic_intr99
),
169 IDTVEC(ioapic_intr100
),
170 IDTVEC(ioapic_intr101
),
171 IDTVEC(ioapic_intr102
),
172 IDTVEC(ioapic_intr103
),
173 IDTVEC(ioapic_intr104
),
174 IDTVEC(ioapic_intr105
),
175 IDTVEC(ioapic_intr106
),
176 IDTVEC(ioapic_intr107
),
177 IDTVEC(ioapic_intr108
),
178 IDTVEC(ioapic_intr109
),
179 IDTVEC(ioapic_intr110
),
180 IDTVEC(ioapic_intr111
),
181 IDTVEC(ioapic_intr112
),
182 IDTVEC(ioapic_intr113
),
183 IDTVEC(ioapic_intr114
),
184 IDTVEC(ioapic_intr115
),
185 IDTVEC(ioapic_intr116
),
186 IDTVEC(ioapic_intr117
),
187 IDTVEC(ioapic_intr118
),
188 IDTVEC(ioapic_intr119
),
189 IDTVEC(ioapic_intr120
),
190 IDTVEC(ioapic_intr121
),
191 IDTVEC(ioapic_intr122
),
192 IDTVEC(ioapic_intr123
),
193 IDTVEC(ioapic_intr124
),
194 IDTVEC(ioapic_intr125
),
195 IDTVEC(ioapic_intr126
),
196 IDTVEC(ioapic_intr127
),
197 IDTVEC(ioapic_intr128
),
198 IDTVEC(ioapic_intr129
),
199 IDTVEC(ioapic_intr130
),
200 IDTVEC(ioapic_intr131
),
201 IDTVEC(ioapic_intr132
),
202 IDTVEC(ioapic_intr133
),
203 IDTVEC(ioapic_intr134
),
204 IDTVEC(ioapic_intr135
),
205 IDTVEC(ioapic_intr136
),
206 IDTVEC(ioapic_intr137
),
207 IDTVEC(ioapic_intr138
),
208 IDTVEC(ioapic_intr139
),
209 IDTVEC(ioapic_intr140
),
210 IDTVEC(ioapic_intr141
),
211 IDTVEC(ioapic_intr142
),
212 IDTVEC(ioapic_intr143
),
213 IDTVEC(ioapic_intr144
),
214 IDTVEC(ioapic_intr145
),
215 IDTVEC(ioapic_intr146
),
216 IDTVEC(ioapic_intr147
),
217 IDTVEC(ioapic_intr148
),
218 IDTVEC(ioapic_intr149
),
219 IDTVEC(ioapic_intr150
),
220 IDTVEC(ioapic_intr151
),
221 IDTVEC(ioapic_intr152
),
222 IDTVEC(ioapic_intr153
),
223 IDTVEC(ioapic_intr154
),
224 IDTVEC(ioapic_intr155
),
225 IDTVEC(ioapic_intr156
),
226 IDTVEC(ioapic_intr157
),
227 IDTVEC(ioapic_intr158
),
228 IDTVEC(ioapic_intr159
),
229 IDTVEC(ioapic_intr160
),
230 IDTVEC(ioapic_intr161
),
231 IDTVEC(ioapic_intr162
),
232 IDTVEC(ioapic_intr163
),
233 IDTVEC(ioapic_intr164
),
234 IDTVEC(ioapic_intr165
),
235 IDTVEC(ioapic_intr166
),
236 IDTVEC(ioapic_intr167
),
237 IDTVEC(ioapic_intr168
),
238 IDTVEC(ioapic_intr169
),
239 IDTVEC(ioapic_intr170
),
240 IDTVEC(ioapic_intr171
),
241 IDTVEC(ioapic_intr172
),
242 IDTVEC(ioapic_intr173
),
243 IDTVEC(ioapic_intr174
),
244 IDTVEC(ioapic_intr175
),
245 IDTVEC(ioapic_intr176
),
246 IDTVEC(ioapic_intr177
),
247 IDTVEC(ioapic_intr178
),
248 IDTVEC(ioapic_intr179
),
249 IDTVEC(ioapic_intr180
),
250 IDTVEC(ioapic_intr181
),
251 IDTVEC(ioapic_intr182
),
252 IDTVEC(ioapic_intr183
),
253 IDTVEC(ioapic_intr184
),
254 IDTVEC(ioapic_intr185
),
255 IDTVEC(ioapic_intr186
),
256 IDTVEC(ioapic_intr187
),
257 IDTVEC(ioapic_intr188
),
258 IDTVEC(ioapic_intr189
),
259 IDTVEC(ioapic_intr190
),
260 IDTVEC(ioapic_intr191
);
262 static inthand_t
*ioapic_intr
[IOAPIC_HWI_VECTORS
] = {
263 &IDTVEC(ioapic_intr0
),
264 &IDTVEC(ioapic_intr1
),
265 &IDTVEC(ioapic_intr2
),
266 &IDTVEC(ioapic_intr3
),
267 &IDTVEC(ioapic_intr4
),
268 &IDTVEC(ioapic_intr5
),
269 &IDTVEC(ioapic_intr6
),
270 &IDTVEC(ioapic_intr7
),
271 &IDTVEC(ioapic_intr8
),
272 &IDTVEC(ioapic_intr9
),
273 &IDTVEC(ioapic_intr10
),
274 &IDTVEC(ioapic_intr11
),
275 &IDTVEC(ioapic_intr12
),
276 &IDTVEC(ioapic_intr13
),
277 &IDTVEC(ioapic_intr14
),
278 &IDTVEC(ioapic_intr15
),
279 &IDTVEC(ioapic_intr16
),
280 &IDTVEC(ioapic_intr17
),
281 &IDTVEC(ioapic_intr18
),
282 &IDTVEC(ioapic_intr19
),
283 &IDTVEC(ioapic_intr20
),
284 &IDTVEC(ioapic_intr21
),
285 &IDTVEC(ioapic_intr22
),
286 &IDTVEC(ioapic_intr23
),
287 &IDTVEC(ioapic_intr24
),
288 &IDTVEC(ioapic_intr25
),
289 &IDTVEC(ioapic_intr26
),
290 &IDTVEC(ioapic_intr27
),
291 &IDTVEC(ioapic_intr28
),
292 &IDTVEC(ioapic_intr29
),
293 &IDTVEC(ioapic_intr30
),
294 &IDTVEC(ioapic_intr31
),
295 &IDTVEC(ioapic_intr32
),
296 &IDTVEC(ioapic_intr33
),
297 &IDTVEC(ioapic_intr34
),
298 &IDTVEC(ioapic_intr35
),
299 &IDTVEC(ioapic_intr36
),
300 &IDTVEC(ioapic_intr37
),
301 &IDTVEC(ioapic_intr38
),
302 &IDTVEC(ioapic_intr39
),
303 &IDTVEC(ioapic_intr40
),
304 &IDTVEC(ioapic_intr41
),
305 &IDTVEC(ioapic_intr42
),
306 &IDTVEC(ioapic_intr43
),
307 &IDTVEC(ioapic_intr44
),
308 &IDTVEC(ioapic_intr45
),
309 &IDTVEC(ioapic_intr46
),
310 &IDTVEC(ioapic_intr47
),
311 &IDTVEC(ioapic_intr48
),
312 &IDTVEC(ioapic_intr49
),
313 &IDTVEC(ioapic_intr50
),
314 &IDTVEC(ioapic_intr51
),
315 &IDTVEC(ioapic_intr52
),
316 &IDTVEC(ioapic_intr53
),
317 &IDTVEC(ioapic_intr54
),
318 &IDTVEC(ioapic_intr55
),
319 &IDTVEC(ioapic_intr56
),
320 &IDTVEC(ioapic_intr57
),
321 &IDTVEC(ioapic_intr58
),
322 &IDTVEC(ioapic_intr59
),
323 &IDTVEC(ioapic_intr60
),
324 &IDTVEC(ioapic_intr61
),
325 &IDTVEC(ioapic_intr62
),
326 &IDTVEC(ioapic_intr63
),
327 &IDTVEC(ioapic_intr64
),
328 &IDTVEC(ioapic_intr65
),
329 &IDTVEC(ioapic_intr66
),
330 &IDTVEC(ioapic_intr67
),
331 &IDTVEC(ioapic_intr68
),
332 &IDTVEC(ioapic_intr69
),
333 &IDTVEC(ioapic_intr70
),
334 &IDTVEC(ioapic_intr71
),
335 &IDTVEC(ioapic_intr72
),
336 &IDTVEC(ioapic_intr73
),
337 &IDTVEC(ioapic_intr74
),
338 &IDTVEC(ioapic_intr75
),
339 &IDTVEC(ioapic_intr76
),
340 &IDTVEC(ioapic_intr77
),
341 &IDTVEC(ioapic_intr78
),
342 &IDTVEC(ioapic_intr79
),
343 &IDTVEC(ioapic_intr80
),
344 &IDTVEC(ioapic_intr81
),
345 &IDTVEC(ioapic_intr82
),
346 &IDTVEC(ioapic_intr83
),
347 &IDTVEC(ioapic_intr84
),
348 &IDTVEC(ioapic_intr85
),
349 &IDTVEC(ioapic_intr86
),
350 &IDTVEC(ioapic_intr87
),
351 &IDTVEC(ioapic_intr88
),
352 &IDTVEC(ioapic_intr89
),
353 &IDTVEC(ioapic_intr90
),
354 &IDTVEC(ioapic_intr91
),
355 &IDTVEC(ioapic_intr92
),
356 &IDTVEC(ioapic_intr93
),
357 &IDTVEC(ioapic_intr94
),
358 &IDTVEC(ioapic_intr95
),
359 &IDTVEC(ioapic_intr96
),
360 &IDTVEC(ioapic_intr97
),
361 &IDTVEC(ioapic_intr98
),
362 &IDTVEC(ioapic_intr99
),
363 &IDTVEC(ioapic_intr100
),
364 &IDTVEC(ioapic_intr101
),
365 &IDTVEC(ioapic_intr102
),
366 &IDTVEC(ioapic_intr103
),
367 &IDTVEC(ioapic_intr104
),
368 &IDTVEC(ioapic_intr105
),
369 &IDTVEC(ioapic_intr106
),
370 &IDTVEC(ioapic_intr107
),
371 &IDTVEC(ioapic_intr108
),
372 &IDTVEC(ioapic_intr109
),
373 &IDTVEC(ioapic_intr110
),
374 &IDTVEC(ioapic_intr111
),
375 &IDTVEC(ioapic_intr112
),
376 &IDTVEC(ioapic_intr113
),
377 &IDTVEC(ioapic_intr114
),
378 &IDTVEC(ioapic_intr115
),
379 &IDTVEC(ioapic_intr116
),
380 &IDTVEC(ioapic_intr117
),
381 &IDTVEC(ioapic_intr118
),
382 &IDTVEC(ioapic_intr119
),
383 &IDTVEC(ioapic_intr120
),
384 &IDTVEC(ioapic_intr121
),
385 &IDTVEC(ioapic_intr122
),
386 &IDTVEC(ioapic_intr123
),
387 &IDTVEC(ioapic_intr124
),
388 &IDTVEC(ioapic_intr125
),
389 &IDTVEC(ioapic_intr126
),
390 &IDTVEC(ioapic_intr127
),
391 &IDTVEC(ioapic_intr128
),
392 &IDTVEC(ioapic_intr129
),
393 &IDTVEC(ioapic_intr130
),
394 &IDTVEC(ioapic_intr131
),
395 &IDTVEC(ioapic_intr132
),
396 &IDTVEC(ioapic_intr133
),
397 &IDTVEC(ioapic_intr134
),
398 &IDTVEC(ioapic_intr135
),
399 &IDTVEC(ioapic_intr136
),
400 &IDTVEC(ioapic_intr137
),
401 &IDTVEC(ioapic_intr138
),
402 &IDTVEC(ioapic_intr139
),
403 &IDTVEC(ioapic_intr140
),
404 &IDTVEC(ioapic_intr141
),
405 &IDTVEC(ioapic_intr142
),
406 &IDTVEC(ioapic_intr143
),
407 &IDTVEC(ioapic_intr144
),
408 &IDTVEC(ioapic_intr145
),
409 &IDTVEC(ioapic_intr146
),
410 &IDTVEC(ioapic_intr147
),
411 &IDTVEC(ioapic_intr148
),
412 &IDTVEC(ioapic_intr149
),
413 &IDTVEC(ioapic_intr150
),
414 &IDTVEC(ioapic_intr151
),
415 &IDTVEC(ioapic_intr152
),
416 &IDTVEC(ioapic_intr153
),
417 &IDTVEC(ioapic_intr154
),
418 &IDTVEC(ioapic_intr155
),
419 &IDTVEC(ioapic_intr156
),
420 &IDTVEC(ioapic_intr157
),
421 &IDTVEC(ioapic_intr158
),
422 &IDTVEC(ioapic_intr159
),
423 &IDTVEC(ioapic_intr160
),
424 &IDTVEC(ioapic_intr161
),
425 &IDTVEC(ioapic_intr162
),
426 &IDTVEC(ioapic_intr163
),
427 &IDTVEC(ioapic_intr164
),
428 &IDTVEC(ioapic_intr165
),
429 &IDTVEC(ioapic_intr166
),
430 &IDTVEC(ioapic_intr167
),
431 &IDTVEC(ioapic_intr168
),
432 &IDTVEC(ioapic_intr169
),
433 &IDTVEC(ioapic_intr170
),
434 &IDTVEC(ioapic_intr171
),
435 &IDTVEC(ioapic_intr172
),
436 &IDTVEC(ioapic_intr173
),
437 &IDTVEC(ioapic_intr174
),
438 &IDTVEC(ioapic_intr175
),
439 &IDTVEC(ioapic_intr176
),
440 &IDTVEC(ioapic_intr177
),
441 &IDTVEC(ioapic_intr178
),
442 &IDTVEC(ioapic_intr179
),
443 &IDTVEC(ioapic_intr180
),
444 &IDTVEC(ioapic_intr181
),
445 &IDTVEC(ioapic_intr182
),
446 &IDTVEC(ioapic_intr183
),
447 &IDTVEC(ioapic_intr184
),
448 &IDTVEC(ioapic_intr185
),
449 &IDTVEC(ioapic_intr186
),
450 &IDTVEC(ioapic_intr187
),
451 &IDTVEC(ioapic_intr188
),
452 &IDTVEC(ioapic_intr189
),
453 &IDTVEC(ioapic_intr190
),
454 &IDTVEC(ioapic_intr191
)
457 #define IOAPIC_HWI_SYSCALL (IDT_OFFSET_SYSCALL - IDT_OFFSET)
460 * NOTE: Initialized before VM so cannot use kmalloc() for this array.
462 static struct ioapic_irqmap
{
463 int im_type
; /* IOAPIC_IMT_ */
464 enum intr_trigger im_trig
;
465 enum intr_polarity im_pola
;
468 uint32_t im_flags
; /* IOAPIC_IMF_ */
469 } ioapic_irqmaps
[MAXCPU
][IOAPIC_HWI_VECTORS
];
471 static struct lwkt_token ioapic_irqmap_tok
=
472 LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token
);
474 #define IOAPIC_IMT_UNUSED 0
475 #define IOAPIC_IMT_RESERVED 1
476 #define IOAPIC_IMT_LEGACY 2
477 #define IOAPIC_IMT_SYSCALL 3
478 #define IOAPIC_IMT_MSI 4
479 #define IOAPIC_IMT_MSIX 5
481 #define IOAPIC_IMT_ISHWI(map) ((map)->im_type != IOAPIC_IMT_RESERVED && \
482 (map)->im_type != IOAPIC_IMT_SYSCALL)
484 #define IOAPIC_IMF_CONF 0x1
486 extern void IOAPIC_INTREN(int);
487 extern void IOAPIC_INTRDIS(int);
489 extern int imcr_present
;
491 static void ioapic_abi_intr_enable(int);
492 static void ioapic_abi_intr_disable(int);
493 static void ioapic_abi_intr_setup(int, int);
494 static void ioapic_abi_intr_teardown(int);
496 static void ioapic_abi_legacy_intr_config(int,
497 enum intr_trigger
, enum intr_polarity
);
498 static int ioapic_abi_legacy_intr_cpuid(int);
499 static int ioapic_abi_legacy_intr_find(int,
500 enum intr_trigger
, enum intr_polarity
);
501 static int ioapic_abi_legacy_intr_find_bygsi(int,
502 enum intr_trigger
, enum intr_polarity
);
504 static int ioapic_abi_msi_alloc(int [], int, int);
505 static void ioapic_abi_msi_release(const int [], int, int);
506 static void ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int);
507 static int ioapic_abi_msix_alloc(int *, int);
508 static void ioapic_abi_msix_release(int, int);
510 static int ioapic_abi_msi_alloc_intern(int, const char *,
512 static void ioapic_abi_msi_release_intern(int, const char *,
513 const int [], int, int);
515 static void ioapic_abi_finalize(void);
516 static void ioapic_abi_cleanup(void);
517 static void ioapic_abi_setdefault(void);
518 static void ioapic_abi_stabilize(void);
519 static void ioapic_abi_initmap(void);
520 static void ioapic_abi_rman_setup(struct rman
*);
522 static int ioapic_abi_gsi_cpuid(int, int);
523 static int ioapic_unused_legacy_irqmap(void);
524 static int ioapic_is_legacy_irqmap_used(int);
526 struct machintr_abi MachIntrABI_IOAPIC
= {
528 .intr_disable
= ioapic_abi_intr_disable
,
529 .intr_enable
= ioapic_abi_intr_enable
,
530 .intr_setup
= ioapic_abi_intr_setup
,
531 .intr_teardown
= ioapic_abi_intr_teardown
,
533 .legacy_intr_config
= ioapic_abi_legacy_intr_config
,
534 .legacy_intr_cpuid
= ioapic_abi_legacy_intr_cpuid
,
535 .legacy_intr_find
= ioapic_abi_legacy_intr_find
,
536 .legacy_intr_find_bygsi
= ioapic_abi_legacy_intr_find_bygsi
,
538 .msi_alloc
= ioapic_abi_msi_alloc
,
539 .msi_release
= ioapic_abi_msi_release
,
540 .msi_map
= ioapic_abi_msi_map
,
541 .msix_alloc
= ioapic_abi_msix_alloc
,
542 .msix_release
= ioapic_abi_msix_release
,
544 .finalize
= ioapic_abi_finalize
,
545 .cleanup
= ioapic_abi_cleanup
,
546 .setdefault
= ioapic_abi_setdefault
,
547 .stabilize
= ioapic_abi_stabilize
,
548 .initmap
= ioapic_abi_initmap
,
549 .rman_setup
= ioapic_abi_rman_setup
552 static int ioapic_abi_extint_irq
= -1;
553 static int ioapic_abi_legacy_irq_max
;
554 static int ioapic_abi_gsi_balance
= 1;
555 static int ioapic_abi_msi_start
; /* NOTE: for testing only */
557 struct ioapic_irqinfo ioapic_irqs
[IOAPIC_HWI_VECTORS
];
560 ioapic_abi_intr_enable(int irq
)
562 const struct ioapic_irqmap
*map
;
564 KASSERT(irq
>= 0 && irq
< IOAPIC_HWI_VECTORS
,
565 ("ioapic enable, invalid irq %d", irq
));
567 map
= &ioapic_irqmaps
[mycpuid
][irq
];
568 KASSERT(IOAPIC_IMT_ISHWI(map
),
569 ("ioapic enable, not hwi irq %d, type %d, cpu%d",
570 irq
, map
->im_type
, mycpuid
));
571 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
578 ioapic_abi_intr_disable(int irq
)
580 const struct ioapic_irqmap
*map
;
582 KASSERT(irq
>= 0 && irq
< IOAPIC_HWI_VECTORS
,
583 ("ioapic disable, invalid irq %d", irq
));
585 map
= &ioapic_irqmaps
[mycpuid
][irq
];
586 KASSERT(IOAPIC_IMT_ISHWI(map
),
587 ("ioapic disable, not hwi irq %d, type %d, cpu%d",
588 irq
, map
->im_type
, mycpuid
));
589 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
596 ioapic_abi_finalize(void)
598 KKASSERT(MachIntrABI
.type
== MACHINTR_IOAPIC
);
599 KKASSERT(ioapic_enable
);
602 * If an IMCR is present, program bit 0 to disconnect the 8259
606 outb(0x22, 0x70); /* select IMCR */
607 outb(0x23, 0x01); /* disconnect 8259 */
612 * This routine is called after physical interrupts are enabled but before
613 * the critical section is released. We need to clean out any interrupts
614 * that had already been posted to the cpu.
617 ioapic_abi_cleanup(void)
619 bzero(mdcpu
->gd_ipending
, sizeof(mdcpu
->gd_ipending
));
622 /* Must never be called */
624 ioapic_abi_stabilize(void)
626 panic("ioapic_stabilize is called");
630 ioapic_abi_intr_setup(int intr
, int flags
)
632 const struct ioapic_irqmap
*map
;
637 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
638 ("ioapic setup, invalid irq %d", intr
));
640 map
= &ioapic_irqmaps
[mycpuid
][intr
];
641 KASSERT(IOAPIC_IMT_ISHWI(map
),
642 ("ioapic setup, not hwi irq %d, type %d, cpu%d",
643 intr
, map
->im_type
, mycpuid
));
644 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
647 KASSERT(ioapic_irqs
[intr
].io_addr
!= NULL
,
648 ("ioapic setup, no GSI information, irq %d", intr
));
653 vector
= IDT_OFFSET
+ intr
;
656 * Now reprogram the vector in the IO APIC. In order to avoid
657 * losing an EOI for a level interrupt, which is vector based,
658 * make sure that the IO APIC is programmed for edge-triggering
659 * first, then reprogrammed with the new vector. This should
664 select
= ioapic_irqs
[intr
].io_idx
;
665 value
= ioapic_read(ioapic_irqs
[intr
].io_addr
, select
);
666 value
|= IOART_INTMSET
;
668 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
669 (value
& ~APIC_TRIGMOD_MASK
));
670 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
671 (value
& ~IOART_INTVEC
) | vector
);
681 ioapic_abi_intr_teardown(int intr
)
683 const struct ioapic_irqmap
*map
;
688 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
689 ("ioapic teardown, invalid irq %d", intr
));
691 map
= &ioapic_irqmaps
[mycpuid
][intr
];
692 KASSERT(IOAPIC_IMT_ISHWI(map
),
693 ("ioapic teardown, not hwi irq %d, type %d, cpu%d",
694 intr
, map
->im_type
, mycpuid
));
695 if (map
->im_type
!= IOAPIC_IMT_LEGACY
)
698 KASSERT(ioapic_irqs
[intr
].io_addr
!= NULL
,
699 ("ioapic teardown, no GSI information, irq %d", intr
));
705 * Teardown an interrupt vector. The vector should already be
706 * installed in the cpu's IDT, but make sure.
708 IOAPIC_INTRDIS(intr
);
710 vector
= IDT_OFFSET
+ intr
;
713 * In order to avoid losing an EOI for a level interrupt, which
714 * is vector based, make sure that the IO APIC is programmed for
715 * edge-triggering first, then reprogrammed with the new vector.
716 * This should clear the IRR bit.
720 select
= ioapic_irqs
[intr
].io_idx
;
721 value
= ioapic_read(ioapic_irqs
[intr
].io_addr
, select
);
723 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
724 (value
& ~APIC_TRIGMOD_MASK
));
725 ioapic_write(ioapic_irqs
[intr
].io_addr
, select
,
726 (value
& ~IOART_INTVEC
) | vector
);
734 ioapic_abi_setdefault(void)
738 for (intr
= 0; intr
< IOAPIC_HWI_VECTORS
; ++intr
) {
739 if (intr
== IOAPIC_HWI_SYSCALL
)
741 setidt_global(IDT_OFFSET
+ intr
, ioapic_intr
[intr
],
742 SDT_SYSIGT
, SEL_KPL
, 0);
747 ioapic_abi_initmap(void)
751 kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance
);
753 kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start
);
754 ioapic_abi_msi_start
&= ~0x1f; /* MUST be 32 aligned */
757 * NOTE: ncpus is not ready yet
759 for (cpu
= 0; cpu
< MAXCPU
; ++cpu
) {
762 for (i
= 0; i
< IOAPIC_HWI_VECTORS
; ++i
) {
763 ioapic_irqmaps
[cpu
][i
].im_gsi
= -1;
764 ioapic_irqmaps
[cpu
][i
].im_msi_base
= -1;
766 ioapic_irqmaps
[cpu
][IOAPIC_HWI_SYSCALL
].im_type
=
772 * Only one CPU can have the legacy IRQ map.
775 ioapic_is_legacy_irqmap_used(int irq
)
779 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
780 if (ioapic_irqmaps
[cpu
][irq
].im_type
!= IOAPIC_IMT_UNUSED
)
787 ioapic_unused_legacy_irqmap(void)
791 for (i
= ISA_IRQ_CNT
; i
< IOAPIC_HWI_VECTORS
; ++i
) {
792 if (i
== acpi_sci_irqno())
794 if (!ioapic_is_legacy_irqmap_used(i
))
801 ioapic_set_legacy_irqmap(int irq
, int gsi
, enum intr_trigger trig
,
802 enum intr_polarity pola
)
804 struct ioapic_irqinfo
*info
;
805 struct ioapic_irqmap
*map
;
809 KKASSERT(trig
== INTR_TRIGGER_EDGE
|| trig
== INTR_TRIGGER_LEVEL
);
810 KKASSERT(pola
== INTR_POLARITY_HIGH
|| pola
== INTR_POLARITY_LOW
);
813 if (irq
>= IOAPIC_HWI_VECTORS
) {
815 * Some BIOSes seem to assume that all 256 IDT vectors
816 * could be used, while we limit the available IDT
817 * vectors to 192; find an unused IRQ for this GSI.
819 irq
= ioapic_unused_legacy_irqmap();
821 kprintf("failed to find unused irq for gsi %d, "
826 KKASSERT(irq
< IOAPIC_HWI_VECTORS
);
828 if (ioapic_is_legacy_irqmap_used(irq
)) {
830 * There are so many IOAPICs, that 1:1 mapping
831 * of GSI and IRQ hits SYSCALL entry.
833 irq
= ioapic_unused_legacy_irqmap();
835 kprintf("failed to find unused irq for gsi %d, "
839 KKASSERT(irq
< IOAPIC_HWI_VECTORS
);
842 cpuid
= ioapic_abi_gsi_cpuid(irq
, gsi
);
843 map
= &ioapic_irqmaps
[cpuid
][irq
];
845 if (irq
> ioapic_abi_legacy_irq_max
)
846 ioapic_abi_legacy_irq_max
= irq
;
848 KKASSERT(map
->im_type
== IOAPIC_IMT_UNUSED
);
849 map
->im_type
= IOAPIC_IMT_LEGACY
;
856 kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n",
858 intr_str_trigger(map
->im_trig
),
859 intr_str_polarity(map
->im_pola
));
862 pin
= ioapic_gsi_pin(map
->im_gsi
);
863 ioaddr
= ioapic_gsi_ioaddr(map
->im_gsi
);
865 info
= &ioapic_irqs
[irq
];
869 info
->io_addr
= ioaddr
;
870 info
->io_idx
= IOAPIC_REDTBL
+ (2 * pin
);
871 info
->io_flags
= IOAPIC_IRQI_FLAG_MASKED
;
872 if (map
->im_trig
== INTR_TRIGGER_LEVEL
)
873 info
->io_flags
|= IOAPIC_IRQI_FLAG_LEVEL
;
875 ioapic_pin_setup(ioaddr
, pin
, IDT_OFFSET
+ irq
,
876 map
->im_trig
, map
->im_pola
, cpuid
);
882 ioapic_fixup_legacy_irqmaps(void)
886 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
889 for (i
= 0; i
< ISA_IRQ_CNT
; ++i
) {
890 struct ioapic_irqmap
*map
= &ioapic_irqmaps
[cpu
][i
];
892 if (map
->im_type
== IOAPIC_IMT_UNUSED
) {
893 map
->im_type
= IOAPIC_IMT_RESERVED
;
896 "cpu%d irq %d reserved\n", cpu
, i
);
902 ioapic_abi_legacy_irq_max
+= 1;
904 kprintf("IOAPIC: legacy irq max %d\n",
905 ioapic_abi_legacy_irq_max
);
910 ioapic_abi_legacy_intr_find_bygsi(int gsi
, enum intr_trigger trig
,
911 enum intr_polarity pola
)
916 if (trig
== INTR_TRIGGER_CONFORM
) {
917 KKASSERT(pola
== INTR_POLARITY_CONFORM
);
919 KKASSERT(trig
== INTR_TRIGGER_EDGE
||
920 trig
== INTR_TRIGGER_LEVEL
);
921 KKASSERT(pola
== INTR_POLARITY_HIGH
||
922 pola
== INTR_POLARITY_LOW
);
926 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
929 for (irq
= 0; irq
< ioapic_abi_legacy_irq_max
; ++irq
) {
930 const struct ioapic_irqmap
*map
=
931 &ioapic_irqmaps
[cpu
][irq
];
933 if (map
->im_gsi
== gsi
) {
934 KKASSERT(map
->im_type
== IOAPIC_IMT_LEGACY
);
936 if ((map
->im_flags
& IOAPIC_IMF_CONF
) &&
937 trig
!= INTR_TRIGGER_CONFORM
&&
938 pola
!= INTR_POLARITY_CONFORM
) {
939 if (map
->im_trig
!= trig
||
940 map
->im_pola
!= pola
)
951 ioapic_abi_legacy_intr_find(int irq
, enum intr_trigger trig
,
952 enum intr_polarity pola
)
957 if (trig
== INTR_TRIGGER_CONFORM
) {
958 KKASSERT(pola
== INTR_POLARITY_CONFORM
);
960 KKASSERT(trig
== INTR_TRIGGER_EDGE
||
961 trig
== INTR_TRIGGER_LEVEL
);
962 KKASSERT(pola
== INTR_POLARITY_HIGH
||
963 pola
== INTR_POLARITY_LOW
);
967 if (irq
< 0 || irq
>= ioapic_abi_legacy_irq_max
)
970 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
971 const struct ioapic_irqmap
*map
= &ioapic_irqmaps
[cpu
][irq
];
973 if (map
->im_type
== IOAPIC_IMT_LEGACY
) {
974 if ((map
->im_flags
& IOAPIC_IMF_CONF
) &&
975 trig
!= INTR_TRIGGER_CONFORM
&&
976 pola
!= INTR_POLARITY_CONFORM
) {
977 if (map
->im_trig
!= trig
||
978 map
->im_pola
!= pola
)
988 ioapic_abi_legacy_intr_config(int irq
, enum intr_trigger trig
,
989 enum intr_polarity pola
)
991 struct ioapic_irqinfo
*info
;
992 struct ioapic_irqmap
*map
= NULL
;
996 KKASSERT(trig
== INTR_TRIGGER_EDGE
|| trig
== INTR_TRIGGER_LEVEL
);
997 KKASSERT(pola
== INTR_POLARITY_HIGH
|| pola
== INTR_POLARITY_LOW
);
999 KKASSERT(irq
>= 0 && irq
< ioapic_abi_legacy_irq_max
);
1000 for (cpuid
= 0; cpuid
< ncpus
; ++cpuid
) {
1001 map
= &ioapic_irqmaps
[cpuid
][irq
];
1002 if (map
->im_type
== IOAPIC_IMT_LEGACY
)
1005 KKASSERT(cpuid
< ncpus
);
1008 if (map
->im_flags
& IOAPIC_IMF_CONF
) {
1009 if (trig
!= map
->im_trig
) {
1010 panic("ioapic_intr_config: trig %s -> %s",
1011 intr_str_trigger(map
->im_trig
),
1012 intr_str_trigger(trig
));
1014 if (pola
!= map
->im_pola
) {
1015 panic("ioapic_intr_config: pola %s -> %s",
1016 intr_str_polarity(map
->im_pola
),
1017 intr_str_polarity(pola
));
1022 map
->im_flags
|= IOAPIC_IMF_CONF
;
1024 if (trig
== map
->im_trig
&& pola
== map
->im_pola
)
1028 kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n",
1030 intr_str_trigger(map
->im_trig
),
1031 intr_str_polarity(map
->im_pola
),
1032 intr_str_trigger(trig
),
1033 intr_str_polarity(pola
));
1035 map
->im_trig
= trig
;
1036 map
->im_pola
= pola
;
1038 pin
= ioapic_gsi_pin(map
->im_gsi
);
1039 ioaddr
= ioapic_gsi_ioaddr(map
->im_gsi
);
1041 info
= &ioapic_irqs
[irq
];
1045 info
->io_flags
&= ~IOAPIC_IRQI_FLAG_LEVEL
;
1046 if (map
->im_trig
== INTR_TRIGGER_LEVEL
)
1047 info
->io_flags
|= IOAPIC_IRQI_FLAG_LEVEL
;
1049 ioapic_pin_setup(ioaddr
, pin
, IDT_OFFSET
+ irq
,
1050 map
->im_trig
, map
->im_pola
, cpuid
);
1056 ioapic_conf_legacy_extint(int irq
)
1058 struct ioapic_irqinfo
*info
;
1059 struct ioapic_irqmap
*map
;
1061 int pin
, error
, vec
;
1063 /* XXX only irq0 is allowed */
1066 vec
= IDT_OFFSET
+ irq
;
1068 if (ioapic_abi_extint_irq
== irq
)
1070 else if (ioapic_abi_extint_irq
>= 0)
1073 error
= icu_ioapic_extint(irq
, vec
);
1077 /* ExtINT is always targeted to cpu0 */
1078 map
= &ioapic_irqmaps
[0][irq
];
1080 KKASSERT(map
->im_type
== IOAPIC_IMT_RESERVED
||
1081 map
->im_type
== IOAPIC_IMT_LEGACY
);
1082 if (map
->im_type
== IOAPIC_IMT_LEGACY
) {
1083 if (map
->im_flags
& IOAPIC_IMF_CONF
)
1086 ioapic_abi_extint_irq
= irq
;
1088 map
->im_type
= IOAPIC_IMT_LEGACY
;
1089 map
->im_trig
= INTR_TRIGGER_EDGE
;
1090 map
->im_pola
= INTR_POLARITY_HIGH
;
1091 map
->im_flags
= IOAPIC_IMF_CONF
;
1093 map
->im_gsi
= ioapic_extpin_gsi();
1094 KKASSERT(map
->im_gsi
>= 0);
1097 kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n",
1099 intr_str_trigger(map
->im_trig
),
1100 intr_str_polarity(map
->im_pola
));
1103 pin
= ioapic_gsi_pin(map
->im_gsi
);
1104 ioaddr
= ioapic_gsi_ioaddr(map
->im_gsi
);
1106 info
= &ioapic_irqs
[irq
];
1110 info
->io_addr
= ioaddr
;
1111 info
->io_idx
= IOAPIC_REDTBL
+ (2 * pin
);
1112 info
->io_flags
= IOAPIC_IRQI_FLAG_MASKED
;
1114 ioapic_extpin_setup(ioaddr
, pin
, vec
);
1122 ioapic_abi_legacy_intr_cpuid(int irq
)
1124 const struct ioapic_irqmap
*map
= NULL
;
1127 KKASSERT(irq
>= 0 && irq
< ioapic_abi_legacy_irq_max
);
1129 for (cpuid
= 0; cpuid
< ncpus
; ++cpuid
) {
1130 map
= &ioapic_irqmaps
[cpuid
][irq
];
1131 if (map
->im_type
== IOAPIC_IMT_LEGACY
)
1135 /* XXX some drivers tries to peek at reserved IRQs */
1136 for (cpuid
= 0; cpuid
< ncpus
; ++cpuid
) {
1137 map
= &ioapic_irqmaps
[cpuid
][irq
];
1138 KKASSERT(map
->im_type
== IOAPIC_IMT_RESERVED
);
1144 ioapic_abi_gsi_cpuid(int irq
, int gsi
)
1151 if (irq
== 0 || gsi
== 0) {
1154 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n",
1160 if (irq
>= 0 && irq
== acpi_sci_irqno()) {
1162 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n",
1168 ksnprintf(envpath
, sizeof(envpath
), "hw.ioapic.gsi.%d.cpu", gsi
);
1169 kgetenv_int(envpath
, &cpuid
);
1172 if (!ioapic_abi_gsi_balance
) {
1173 if (irq
>= 0 && bootverbose
) {
1174 kprintf("IOAPIC: irq %d, gsi %d -> cpu0 "
1175 "(fixed)\n", irq
, gsi
);
1180 cpuid
= gsi
% ncpus
;
1181 if (irq
>= 0 && bootverbose
) {
1182 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n",
1185 } else if (cpuid
>= ncpus
) {
1187 if (irq
>= 0 && bootverbose
) {
1188 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n",
1192 if (irq
>= 0 && bootverbose
) {
1193 kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n",
1201 ioapic_abi_rman_setup(struct rman
*rm
)
1205 KASSERT(rm
->rm_cpuid
>= 0 && rm
->rm_cpuid
< MAXCPU
,
1206 ("invalid rman cpuid %d", rm
->rm_cpuid
));
1209 for (i
= 0; i
< IOAPIC_HWI_VECTORS
; ++i
) {
1210 const struct ioapic_irqmap
*map
=
1211 &ioapic_irqmaps
[rm
->rm_cpuid
][i
];
1214 if (IOAPIC_IMT_ISHWI(map
))
1217 if (IOAPIC_IMT_ISHWI(map
)) {
1222 kprintf("IOAPIC: rman cpu%d %d - %d\n",
1223 rm
->rm_cpuid
, start
, end
);
1225 if (rman_manage_region(rm
, start
, end
)) {
1226 panic("rman_manage_region"
1227 "(cpu%d %d - %d)", rm
->rm_cpuid
,
1237 kprintf("IOAPIC: rman cpu%d %d - %d\n",
1238 rm
->rm_cpuid
, start
, end
);
1240 if (rman_manage_region(rm
, start
, end
)) {
1241 panic("rman_manage_region(cpu%d %d - %d)",
1242 rm
->rm_cpuid
, start
, end
);
1248 ioapic_abi_msi_alloc_intern(int type
, const char *desc
,
1249 int intrs
[], int count
, int cpuid
)
1253 KASSERT(cpuid
>= 0 && cpuid
< ncpus
,
1254 ("invalid cpuid %d", cpuid
));
1256 KASSERT(count
> 0 && count
<= 32, ("invalid count %d", count
));
1257 KASSERT(powerof2(count
), ("count %d is not power of 2", count
));
1259 lwkt_gettoken(&ioapic_irqmap_tok
);
1263 * Since IDT_OFFSET is 32, which is the maximum valid 'count',
1264 * we do not need to find out the first properly aligned
1269 for (i
= ioapic_abi_msi_start
; i
< IOAPIC_HWI_VECTORS
; i
+= count
) {
1272 if (ioapic_irqmaps
[cpuid
][i
].im_type
!= IOAPIC_IMT_UNUSED
)
1275 for (j
= 1; j
< count
; ++j
) {
1276 if (ioapic_irqmaps
[cpuid
][i
+ j
].im_type
!=
1283 for (j
= 0; j
< count
; ++j
) {
1284 struct ioapic_irqmap
*map
;
1287 map
= &ioapic_irqmaps
[cpuid
][intr
];
1288 KASSERT(map
->im_msi_base
< 0,
1289 ("intr %d, stale %s-base %d",
1290 intr
, desc
, map
->im_msi_base
));
1292 map
->im_type
= type
;
1293 map
->im_msi_base
= i
;
1296 msi_setup(intr
, cpuid
);
1299 kprintf("alloc %s intr %d on cpu%d\n",
1307 lwkt_reltoken(&ioapic_irqmap_tok
);
1313 ioapic_abi_msi_release_intern(int type
, const char *desc
,
1314 const int intrs
[], int count
, int cpuid
)
1316 int i
, msi_base
= -1, intr_next
= -1, mask
;
1318 KASSERT(cpuid
>= 0 && cpuid
< ncpus
,
1319 ("invalid cpuid %d", cpuid
));
1321 KASSERT(count
> 0 && count
<= 32, ("invalid count %d", count
));
1324 KASSERT((count
& mask
) == 0, ("count %d is not power of 2", count
));
1326 lwkt_gettoken(&ioapic_irqmap_tok
);
1328 for (i
= 0; i
< count
; ++i
) {
1329 struct ioapic_irqmap
*map
;
1330 int intr
= intrs
[i
];
1332 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
1333 ("invalid intr %d", intr
));
1335 map
= &ioapic_irqmaps
[cpuid
][intr
];
1336 KASSERT(map
->im_type
== type
,
1337 ("trying to release non-%s intr %d, type %d", desc
,
1338 intr
, map
->im_type
));
1339 KASSERT(map
->im_msi_base
>= 0 && map
->im_msi_base
<= intr
,
1340 ("intr %d, invalid %s-base %d", intr
, desc
,
1342 KASSERT((map
->im_msi_base
& mask
) == 0,
1343 ("intr %d, %s-base %d is not properly aligned %d",
1344 intr
, desc
, map
->im_msi_base
, count
));
1347 msi_base
= map
->im_msi_base
;
1349 KASSERT(map
->im_msi_base
== msi_base
,
1350 ("intr %d, inconsistent %s-base, "
1352 intr
, desc
, msi_base
, map
->im_msi_base
));
1355 if (intr_next
< intr
)
1358 map
->im_type
= IOAPIC_IMT_UNUSED
;
1359 map
->im_msi_base
= -1;
1362 kprintf("release %s intr %d on cpu%d\n",
1367 KKASSERT(intr_next
> 0);
1368 KKASSERT(msi_base
>= 0);
1371 if (intr_next
< IOAPIC_HWI_VECTORS
) {
1372 const struct ioapic_irqmap
*map
=
1373 &ioapic_irqmaps
[cpuid
][intr_next
];
1375 if (map
->im_type
== type
) {
1376 KASSERT(map
->im_msi_base
!= msi_base
,
1377 ("more than %d %s was allocated", count
, desc
));
1381 lwkt_reltoken(&ioapic_irqmap_tok
);
1385 ioapic_abi_msi_alloc(int intrs
[], int count
, int cpuid
)
1387 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI
, "MSI",
1388 intrs
, count
, cpuid
);
1392 ioapic_abi_msi_release(const int intrs
[], int count
, int cpuid
)
1394 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI
, "MSI",
1395 intrs
, count
, cpuid
);
1399 ioapic_abi_msix_alloc(int *intr
, int cpuid
)
1401 return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX
, "MSI-X",
1406 ioapic_abi_msix_release(int intr
, int cpuid
)
1408 ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX
, "MSI-X",
1413 ioapic_abi_msi_map(int intr
, uint64_t *addr
, uint32_t *data
, int cpuid
)
1415 const struct ioapic_irqmap
*map
;
1417 KASSERT(cpuid
>= 0 && cpuid
< ncpus
,
1418 ("invalid cpuid %d", cpuid
));
1420 KASSERT(intr
>= 0 && intr
< IOAPIC_HWI_VECTORS
,
1421 ("invalid intr %d", intr
));
1423 lwkt_gettoken(&ioapic_irqmap_tok
);
1425 map
= &ioapic_irqmaps
[cpuid
][intr
];
1426 KASSERT(map
->im_type
== IOAPIC_IMT_MSI
||
1427 map
->im_type
== IOAPIC_IMT_MSIX
,
1428 ("trying to map non-MSI/MSI-X intr %d, type %d", intr
, map
->im_type
));
1429 KASSERT(map
->im_msi_base
>= 0 && map
->im_msi_base
<= intr
,
1430 ("intr %d, invalid %s-base %d", intr
,
1431 map
->im_type
== IOAPIC_IMT_MSI
? "MSI" : "MSI-X",
1434 msi_map(map
->im_msi_base
, addr
, data
, cpuid
);
1437 kprintf("map %s intr %d on cpu%d\n",
1438 map
->im_type
== IOAPIC_IMT_MSI
? "MSI" : "MSI-X",
1442 lwkt_reltoken(&ioapic_irqmap_tok
);