?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/amd-iommu: Fix boot crash with hidden PCI devices
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / arch / x86 / kernel / amd_iommu.c
blobe7115c818ba9e24ee4589e7450b46564aeaf4897
1 /*
2 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/pci.h>
21 #include <linux/bitmap.h>
22 #include <linux/slab.h>
23 #include <linux/debugfs.h>
24 #include <linux/scatterlist.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/iommu-helper.h>
27 #include <linux/iommu.h>
28 #include <asm/proto.h>
29 #include <asm/iommu.h>
30 #include <asm/gart.h>
31 #include <asm/dma.h>
32 #include <asm/amd_iommu_proto.h>
33 #include <asm/amd_iommu_types.h>
34 #include <asm/amd_iommu.h>
35
36 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
37
38 #define EXIT_LOOP_COUNT 10000000
39
40 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
41
42 /* A list of preallocated protection domains */
43 static LIST_HEAD(iommu_pd_list);
44 static DEFINE_SPINLOCK(iommu_pd_list_lock);
45
46 /*
47 * Domain for untranslated devices - only allocated
48 * if iommu=pt passed on kernel cmd line.
49 */
50 static struct protection_domain *pt_domain;
51
52 static struct iommu_ops amd_iommu_ops;
53
54 /*
55 * general struct to manage commands send to an IOMMU
56 */
57 struct iommu_cmd {
58 u32 data[4];
59 };
60
61 static void reset_iommu_command_buffer(struct amd_iommu *iommu);
62 static void update_domain(struct protection_domain *domain);
63
64 /****************************************************************************
65 *
66 * Helper functions
67 *
68 ****************************************************************************/
69
70 static inline u16 get_device_id(struct device *dev)
71 {
72 struct pci_dev *pdev = to_pci_dev(dev);
73
74 return calc_devid(pdev->bus->number, pdev->devfn);
75 }
76
77 static struct iommu_dev_data *get_dev_data(struct device *dev)
78 {
79 return dev->archdata.iommu;
80 }
81
82 /*
83 * In this function the list of preallocated protection domains is traversed to
84 * find the domain for a specific device
85 */
86 static struct dma_ops_domain *find_protection_domain(u16 devid)
87 {
88 struct dma_ops_domain *entry, *ret = NULL;
89 unsigned long flags;
90 u16 alias = amd_iommu_alias_table[devid];
91
92 if (list_empty(&iommu_pd_list))
93 return NULL;
94
95 spin_lock_irqsave(&iommu_pd_list_lock, flags);
96
97 list_for_each_entry(entry, &iommu_pd_list, list) {
98 if (entry->target_dev == devid ||
99 entry->target_dev == alias) {
100 ret = entry;
101 break;
102 }
103 }
104
105 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
106
107 return ret;
108 }
109
110 /*
111 * This function checks if the driver got a valid device from the caller to
112 * avoid dereferencing invalid pointers.
113 */
114 static bool check_device(struct device *dev)
115 {
116 u16 devid;
117
118 if (!dev || !dev->dma_mask)
119 return false;
120
121 /* No device or no PCI device */
122 if (dev->bus != &pci_bus_type)
123 return false;
124
125 devid = get_device_id(dev);
126
127 /* Out of our scope? */
128 if (devid > amd_iommu_last_bdf)
129 return false;
130
131 if (amd_iommu_rlookup_table[devid] == NULL)
132 return false;
133
134 return true;
135 }
136
137 static int iommu_init_device(struct device *dev)
138 {
139 struct iommu_dev_data *dev_data;
140 struct pci_dev *pdev;
141 u16 devid, alias;
142
143 if (dev->archdata.iommu)
144 return 0;
145
146 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
147 if (!dev_data)
148 return -ENOMEM;
149
150 dev_data->dev = dev;
151
152 devid = get_device_id(dev);
153 alias = amd_iommu_alias_table[devid];
154 pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff);
155 if (pdev)
156 dev_data->alias = &pdev->dev;
157 else {
158 kfree(dev_data);
159 return -ENOTSUPP;
160 }
161
162 atomic_set(&dev_data->bind, 0);
163
164 dev->archdata.iommu = dev_data;
165
166
167 return 0;
168 }
169
170 static void iommu_ignore_device(struct device *dev)
171 {
172 u16 devid, alias;
173
174 devid = get_device_id(dev);
175 alias = amd_iommu_alias_table[devid];
176
177 memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
178 memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
179
180 amd_iommu_rlookup_table[devid] = NULL;
181 amd_iommu_rlookup_table[alias] = NULL;
182 }
183
184 static void iommu_uninit_device(struct device *dev)
185 {
186 kfree(dev->archdata.iommu);
187 }
188
189 void __init amd_iommu_uninit_devices(void)
190 {
191 struct pci_dev *pdev = NULL;
192
193 for_each_pci_dev(pdev) {
194
195 if (!check_device(&pdev->dev))
196 continue;
197
198 iommu_uninit_device(&pdev->dev);
199 }
200 }
201
202 int __init amd_iommu_init_devices(void)
203 {
204 struct pci_dev *pdev = NULL;
205 int ret = 0;
206
207 for_each_pci_dev(pdev) {
208
209 if (!check_device(&pdev->dev))
210 continue;
211
212 ret = iommu_init_device(&pdev->dev);
213 if (ret == -ENOTSUPP)
214 iommu_ignore_device(&pdev->dev);
215 else if (ret)
216 goto out_free;
217 }
218
219 return 0;
220
221 out_free:
222
223 amd_iommu_uninit_devices();
224
225 return ret;
226 }
227 #ifdef CONFIG_AMD_IOMMU_STATS
228
229 /*
230 * Initialization code for statistics collection
231 */
232
233 DECLARE_STATS_COUNTER(compl_wait);
234 DECLARE_STATS_COUNTER(cnt_map_single);
235 DECLARE_STATS_COUNTER(cnt_unmap_single);
236 DECLARE_STATS_COUNTER(cnt_map_sg);
237 DECLARE_STATS_COUNTER(cnt_unmap_sg);
238 DECLARE_STATS_COUNTER(cnt_alloc_coherent);
239 DECLARE_STATS_COUNTER(cnt_free_coherent);
240 DECLARE_STATS_COUNTER(cross_page);
241 DECLARE_STATS_COUNTER(domain_flush_single);
242 DECLARE_STATS_COUNTER(domain_flush_all);
243 DECLARE_STATS_COUNTER(alloced_io_mem);
244 DECLARE_STATS_COUNTER(total_map_requests);
245
246 static struct dentry *stats_dir;
247 static struct dentry *de_fflush;
248
249 static void amd_iommu_stats_add(struct __iommu_counter *cnt)
250 {
251 if (stats_dir == NULL)
252 return;
253
254 cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
255 &cnt->value);
256 }
257
258 static void amd_iommu_stats_init(void)
259 {
260 stats_dir = debugfs_create_dir("amd-iommu", NULL);
261 if (stats_dir == NULL)
262 return;
263
264 de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
265 (u32 *)&amd_iommu_unmap_flush);
266
267 amd_iommu_stats_add(&compl_wait);
268 amd_iommu_stats_add(&cnt_map_single);
269 amd_iommu_stats_add(&cnt_unmap_single);
270 amd_iommu_stats_add(&cnt_map_sg);
271 amd_iommu_stats_add(&cnt_unmap_sg);
272 amd_iommu_stats_add(&cnt_alloc_coherent);
273 amd_iommu_stats_add(&cnt_free_coherent);
274 amd_iommu_stats_add(&cross_page);
275 amd_iommu_stats_add(&domain_flush_single);
276 amd_iommu_stats_add(&domain_flush_all);
277 amd_iommu_stats_add(&alloced_io_mem);
278 amd_iommu_stats_add(&total_map_requests);
279 }
280
281 #endif
282
283 /****************************************************************************
284 *
285 * Interrupt handling functions
286 *
287 ****************************************************************************/
288
289 static void dump_dte_entry(u16 devid)
290 {
291 int i;
292
293 for (i = 0; i < 8; ++i)
294 pr_err("AMD-Vi: DTE[%d]: %08x\n", i,
295 amd_iommu_dev_table[devid].data[i]);
296 }
297
298 static void dump_command(unsigned long phys_addr)
299 {
300 struct iommu_cmd *cmd = phys_to_virt(phys_addr);
301 int i;
302
303 for (i = 0; i < 4; ++i)
304 pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
305 }
306
307 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
308 {
309 u32 *event = __evt;
310 int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
311 int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
312 int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
313 int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
314 u64 address = (u64)(((u64)event[3]) << 32) | event[2];
315
316 printk(KERN_ERR "AMD-Vi: Event logged [");
317
318 switch (type) {
319 case EVENT_TYPE_ILL_DEV:
320 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
321 "address=0x%016llx flags=0x%04x]\n",
322 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
323 address, flags);
324 dump_dte_entry(devid);
325 break;
326 case EVENT_TYPE_IO_FAULT:
327 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
328 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
329 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
330 domid, address, flags);
331 break;
332 case EVENT_TYPE_DEV_TAB_ERR:
333 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
334 "address=0x%016llx flags=0x%04x]\n",
335 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
336 address, flags);
337 break;
338 case EVENT_TYPE_PAGE_TAB_ERR:
339 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
340 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
341 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
342 domid, address, flags);
343 break;
344 case EVENT_TYPE_ILL_CMD:
345 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
346 iommu->reset_in_progress = true;
347 reset_iommu_command_buffer(iommu);
348 dump_command(address);
349 break;
350 case EVENT_TYPE_CMD_HARD_ERR:
351 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
352 "flags=0x%04x]\n", address, flags);
353 break;
354 case EVENT_TYPE_IOTLB_INV_TO:
355 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
356 "address=0x%016llx]\n",
357 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
358 address);
359 break;
360 case EVENT_TYPE_INV_DEV_REQ:
361 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
362 "address=0x%016llx flags=0x%04x]\n",
363 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
364 address, flags);
365 break;
366 default:
367 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
368 }
369 }
370
371 static void iommu_poll_events(struct amd_iommu *iommu)
372 {
373 u32 head, tail;
374 unsigned long flags;
375
376 spin_lock_irqsave(&iommu->lock, flags);
377
378 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
379 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
380
381 while (head != tail) {
382 iommu_print_event(iommu, iommu->evt_buf + head);
383 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
384 }
385
386 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
387
388 spin_unlock_irqrestore(&iommu->lock, flags);
389 }
390
391 irqreturn_t amd_iommu_int_handler(int irq, void *data)
392 {
393 struct amd_iommu *iommu;
394
395 for_each_iommu(iommu)
396 iommu_poll_events(iommu);
397
398 return IRQ_HANDLED;
399 }
400
401 /****************************************************************************
402 *
403 * IOMMU command queuing functions
404 *
405 ****************************************************************************/
406
407 /*
408 * Writes the command to the IOMMUs command buffer and informs the
409 * hardware about the new command. Must be called with iommu->lock held.
410 */
411 static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
412 {
413 u32 tail, head;
414 u8 *target;
415
416 WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
417 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
418 target = iommu->cmd_buf + tail;
419 memcpy_toio(target, cmd, sizeof(*cmd));
420 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
421 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
422 if (tail == head)
423 return -ENOMEM;
424 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
425
426 return 0;
427 }
428
429 /*
430 * General queuing function for commands. Takes iommu->lock and calls
431 * __iommu_queue_command().
432 */
433 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
434 {
435 unsigned long flags;
436 int ret;
437
438 spin_lock_irqsave(&iommu->lock, flags);
439 ret = __iommu_queue_command(iommu, cmd);
440 if (!ret)
441 iommu->need_sync = true;
442 spin_unlock_irqrestore(&iommu->lock, flags);
443
444 return ret;
445 }
446
447 /*
448 * This function waits until an IOMMU has completed a completion
449 * wait command
450 */
451 static void __iommu_wait_for_completion(struct amd_iommu *iommu)
452 {
453 int ready = 0;
454 unsigned status = 0;
455 unsigned long i = 0;
456
457 INC_STATS_COUNTER(compl_wait);
458
459 while (!ready && (i < EXIT_LOOP_COUNT)) {
460 ++i;
461 /* wait for the bit to become one */
462 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
463 ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
464 }
465
466 /* set bit back to zero */
467 status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
468 writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
469
470 if (unlikely(i == EXIT_LOOP_COUNT))
471 iommu->reset_in_progress = true;
472 }
473
474 /*
475 * This function queues a completion wait command into the command
476 * buffer of an IOMMU
477 */
478 static int __iommu_completion_wait(struct amd_iommu *iommu)
479 {
480 struct iommu_cmd cmd;
481
482 memset(&cmd, 0, sizeof(cmd));
483 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
484 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
485
486 return __iommu_queue_command(iommu, &cmd);
487 }
488
489 /*
490 * This function is called whenever we need to ensure that the IOMMU has
491 * completed execution of all commands we sent. It sends a
492 * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
493 * us about that by writing a value to a physical address we pass with
494 * the command.
495 */
496 static int iommu_completion_wait(struct amd_iommu *iommu)
497 {
498 int ret = 0;
499 unsigned long flags;
500
501 spin_lock_irqsave(&iommu->lock, flags);
502
503 if (!iommu->need_sync)
504 goto out;
505
506 ret = __iommu_completion_wait(iommu);
507
508 iommu->need_sync = false;
509
510 if (ret)
511 goto out;
512
513 __iommu_wait_for_completion(iommu);
514
515 out:
516 spin_unlock_irqrestore(&iommu->lock, flags);
517
518 if (iommu->reset_in_progress)
519 reset_iommu_command_buffer(iommu);
520
521 return 0;
522 }
523
524 static void iommu_flush_complete(struct protection_domain *domain)
525 {
526 int i;
527
528 for (i = 0; i < amd_iommus_present; ++i) {
529 if (!domain->dev_iommu[i])
530 continue;
531
532 /*
533 * Devices of this domain are behind this IOMMU
534 * We need to wait for completion of all commands.
535 */
536 iommu_completion_wait(amd_iommus[i]);
537 }
538 }
539
540 /*
541 * Command send function for invalidating a device table entry
542 */
543 static int iommu_flush_device(struct device *dev)
544 {
545 struct amd_iommu *iommu;
546 struct iommu_cmd cmd;
547 u16 devid;
548
549 devid = get_device_id(dev);
550 iommu = amd_iommu_rlookup_table[devid];
551
552 /* Build command */
553 memset(&cmd, 0, sizeof(cmd));
554 CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
555 cmd.data[0] = devid;
556
557 return iommu_queue_command(iommu, &cmd);
558 }
559
560 static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
561 u16 domid, int pde, int s)
562 {
563 memset(cmd, 0, sizeof(*cmd));
564 address &= PAGE_MASK;
565 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
566 cmd->data[1] |= domid;
567 cmd->data[2] = lower_32_bits(address);
568 cmd->data[3] = upper_32_bits(address);
569 if (s) /* size bit - we flush more than one 4kb page */
570 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
571 if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
572 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
573 }
574
575 /*
576 * Generic command send function for invalidaing TLB entries
577 */
578 static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
579 u64 address, u16 domid, int pde, int s)
580 {
581 struct iommu_cmd cmd;
582 int ret;
583
584 __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
585
586 ret = iommu_queue_command(iommu, &cmd);
587
588 return ret;
589 }
590
591 /*
592 * TLB invalidation function which is called from the mapping functions.
593 * It invalidates a single PTE if the range to flush is within a single
594 * page. Otherwise it flushes the whole TLB of the IOMMU.
595 */
596 static void __iommu_flush_pages(struct protection_domain *domain,
597 u64 address, size_t size, int pde)
598 {
599 int s = 0, i;
600 unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE);
601
602 address &= PAGE_MASK;
603
604 if (pages > 1) {
605 /*
606 * If we have to flush more than one page, flush all
607 * TLB entries for this domain
608 */
609 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
610 s = 1;
611 }
612
613
614 for (i = 0; i < amd_iommus_present; ++i) {
615 if (!domain->dev_iommu[i])
616 continue;
617
618 /*
619 * Devices of this domain are behind this IOMMU
620 * We need a TLB flush
621 */
622 iommu_queue_inv_iommu_pages(amd_iommus[i], address,
623 domain->id, pde, s);
624 }
625
626 return;
627 }
628
629 static void iommu_flush_pages(struct protection_domain *domain,
630 u64 address, size_t size)
631 {
632 __iommu_flush_pages(domain, address, size, 0);
633 }
634
635 /* Flush the whole IO/TLB for a given protection domain */
636 static void iommu_flush_tlb(struct protection_domain *domain)
637 {
638 __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
639 }
640
641 /* Flush the whole IO/TLB for a given protection domain - including PDE */
642 static void iommu_flush_tlb_pde(struct protection_domain *domain)
643 {
644 __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
645 }
646
647
648 /*
649 * This function flushes the DTEs for all devices in domain
650 */
651 static void iommu_flush_domain_devices(struct protection_domain *domain)
652 {
653 struct iommu_dev_data *dev_data;
654 unsigned long flags;
655
656 spin_lock_irqsave(&domain->lock, flags);
657
658 list_for_each_entry(dev_data, &domain->dev_list, list)
659 iommu_flush_device(dev_data->dev);
660
661 spin_unlock_irqrestore(&domain->lock, flags);
662 }
663
664 static void iommu_flush_all_domain_devices(void)
665 {
666 struct protection_domain *domain;
667 unsigned long flags;
668
669 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
670
671 list_for_each_entry(domain, &amd_iommu_pd_list, list) {
672 iommu_flush_domain_devices(domain);
673 iommu_flush_complete(domain);
674 }
675
676 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
677 }
678
679 void amd_iommu_flush_all_devices(void)
680 {
681 iommu_flush_all_domain_devices();
682 }
683
684 /*
685 * This function uses heavy locking and may disable irqs for some time. But
686 * this is no issue because it is only called during resume.
687 */
688 void amd_iommu_flush_all_domains(void)
689 {
690 struct protection_domain *domain;
691 unsigned long flags;
692
693 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
694
695 list_for_each_entry(domain, &amd_iommu_pd_list, list) {
696 spin_lock(&domain->lock);
697 iommu_flush_tlb_pde(domain);
698 iommu_flush_complete(domain);
699 spin_unlock(&domain->lock);
700 }
701
702 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
703 }
704
705 static void reset_iommu_command_buffer(struct amd_iommu *iommu)
706 {
707 pr_err("AMD-Vi: Resetting IOMMU command buffer\n");
708
709 if (iommu->reset_in_progress)
710 panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n");
711
712 amd_iommu_reset_cmd_buffer(iommu);
713 amd_iommu_flush_all_devices();
714 amd_iommu_flush_all_domains();
715
716 iommu->reset_in_progress = false;
717 }
718
719 /****************************************************************************
720 *
721 * The functions below are used the create the page table mappings for
722 * unity mapped regions.
723 *
724 ****************************************************************************/
725
726 /*
727 * This function is used to add another level to an IO page table. Adding
728 * another level increases the size of the address space by 9 bits to a size up
729 * to 64 bits.
730 */
731 static bool increase_address_space(struct protection_domain *domain,
732 gfp_t gfp)
733 {
734 u64 *pte;
735
736 if (domain->mode == PAGE_MODE_6_LEVEL)
737 /* address space already 64 bit large */
738 return false;
739
740 pte = (void *)get_zeroed_page(gfp);
741 if (!pte)
742 return false;
743
744 *pte = PM_LEVEL_PDE(domain->mode,
745 virt_to_phys(domain->pt_root));
746 domain->pt_root = pte;
747 domain->mode += 1;
748 domain->updated = true;
749
750 return true;
751 }
752
753 static u64 *alloc_pte(struct protection_domain *domain,
754 unsigned long address,
755 unsigned long page_size,
756 u64 **pte_page,
757 gfp_t gfp)
758 {
759 int level, end_lvl;
760 u64 *pte, *page;
761
762 BUG_ON(!is_power_of_2(page_size));
763
764 while (address > PM_LEVEL_SIZE(domain->mode))
765 increase_address_space(domain, gfp);
766
767 level = domain->mode - 1;
768 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
769 address = PAGE_SIZE_ALIGN(address, page_size);
770 end_lvl = PAGE_SIZE_LEVEL(page_size);
771
772 while (level > end_lvl) {
773 if (!IOMMU_PTE_PRESENT(*pte)) {
774 page = (u64 *)get_zeroed_page(gfp);
775 if (!page)
776 return NULL;
777 *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
778 }
779
780 /* No level skipping support yet */
781 if (PM_PTE_LEVEL(*pte) != level)
782 return NULL;
783
784 level -= 1;
785
786 pte = IOMMU_PTE_PAGE(*pte);
787
788 if (pte_page && level == end_lvl)
789 *pte_page = pte;
790
791 pte = &pte[PM_LEVEL_INDEX(level, address)];
792 }
793
794 return pte;
795 }
796
797 /*
798 * This function checks if there is a PTE for a given dma address. If
799 * there is one, it returns the pointer to it.
800 */
801 static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
802 {
803 int level;
804 u64 *pte;
805
806 if (address > PM_LEVEL_SIZE(domain->mode))
807 return NULL;
808
809 level = domain->mode - 1;
810 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
811
812 while (level > 0) {
813
814 /* Not Present */
815 if (!IOMMU_PTE_PRESENT(*pte))
816 return NULL;
817
818 /* Large PTE */
819 if (PM_PTE_LEVEL(*pte) == 0x07) {
820 unsigned long pte_mask, __pte;
821
822 /*
823 * If we have a series of large PTEs, make
824 * sure to return a pointer to the first one.
825 */
826 pte_mask = PTE_PAGE_SIZE(*pte);
827 pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
828 __pte = ((unsigned long)pte) & pte_mask;
829
830 return (u64 *)__pte;
831 }
832
833 /* No level skipping support yet */
834 if (PM_PTE_LEVEL(*pte) != level)
835 return NULL;
836
837 level -= 1;
838
839 /* Walk to the next level */
840 pte = IOMMU_PTE_PAGE(*pte);
841 pte = &pte[PM_LEVEL_INDEX(level, address)];
842 }
843
844 return pte;
845 }
846
847 /*
848 * Generic mapping functions. It maps a physical address into a DMA
849 * address space. It allocates the page table pages if necessary.
850 * In the future it can be extended to a generic mapping function
851 * supporting all features of AMD IOMMU page tables like level skipping
852 * and full 64 bit address spaces.
853 */
854 static int iommu_map_page(struct protection_domain *dom,
855 unsigned long bus_addr,
856 unsigned long phys_addr,
857 int prot,
858 unsigned long page_size)
859 {
860 u64 __pte, *pte;
861 int i, count;
862
863 if (!(prot & IOMMU_PROT_MASK))
864 return -EINVAL;
865
866 bus_addr = PAGE_ALIGN(bus_addr);
867 phys_addr = PAGE_ALIGN(phys_addr);
868 count = PAGE_SIZE_PTE_COUNT(page_size);
869 pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
870
871 for (i = 0; i < count; ++i)
872 if (IOMMU_PTE_PRESENT(pte[i]))
873 return -EBUSY;
874
875 if (page_size > PAGE_SIZE) {
876 __pte = PAGE_SIZE_PTE(phys_addr, page_size);
877 __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
878 } else
879 __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
880
881 if (prot & IOMMU_PROT_IR)
882 __pte |= IOMMU_PTE_IR;
883 if (prot & IOMMU_PROT_IW)
884 __pte |= IOMMU_PTE_IW;
885
886 for (i = 0; i < count; ++i)
887 pte[i] = __pte;
888
889 update_domain(dom);
890
891 return 0;
892 }
893
894 static unsigned long iommu_unmap_page(struct protection_domain *dom,
895 unsigned long bus_addr,
896 unsigned long page_size)
897 {
898 unsigned long long unmap_size, unmapped;
899 u64 *pte;
900
901 BUG_ON(!is_power_of_2(page_size));
902
903 unmapped = 0;
904
905 while (unmapped < page_size) {
906
907 pte = fetch_pte(dom, bus_addr);
908
909 if (!pte) {
910 /*
911 * No PTE for this address
912 * move forward in 4kb steps
913 */
914 unmap_size = PAGE_SIZE;
915 } else if (PM_PTE_LEVEL(*pte) == 0) {
916 /* 4kb PTE found for this address */
917 unmap_size = PAGE_SIZE;
918 *pte = 0ULL;
919 } else {
920 int count, i;
921
922 /* Large PTE found which maps this address */
923 unmap_size = PTE_PAGE_SIZE(*pte);
924 count = PAGE_SIZE_PTE_COUNT(unmap_size);
925 for (i = 0; i < count; i++)
926 pte[i] = 0ULL;
927 }
928
929 bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
930 unmapped += unmap_size;
931 }
932
933 BUG_ON(!is_power_of_2(unmapped));
934
935 return unmapped;
936 }
937
938 /*
939 * This function checks if a specific unity mapping entry is needed for
940 * this specific IOMMU.
941 */
942 static int iommu_for_unity_map(struct amd_iommu *iommu,
943 struct unity_map_entry *entry)
944 {
945 u16 bdf, i;
946
947 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
948 bdf = amd_iommu_alias_table[i];
949 if (amd_iommu_rlookup_table[bdf] == iommu)
950 return 1;
951 }
952
953 return 0;
954 }
955
956 /*
957 * This function actually applies the mapping to the page table of the
958 * dma_ops domain.
959 */
960 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
961 struct unity_map_entry *e)
962 {
963 u64 addr;
964 int ret;
965
966 for (addr = e->address_start; addr < e->address_end;
967 addr += PAGE_SIZE) {
968 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
969 PAGE_SIZE);
970 if (ret)
971 return ret;
972 /*
973 * if unity mapping is in aperture range mark the page
974 * as allocated in the aperture
975 */
976 if (addr < dma_dom->aperture_size)
977 __set_bit(addr >> PAGE_SHIFT,
978 dma_dom->aperture[0]->bitmap);
979 }
980
981 return 0;
982 }
983
984 /*
985 * Init the unity mappings for a specific IOMMU in the system
986 *
987 * Basically iterates over all unity mapping entries and applies them to
988 * the default domain DMA of that IOMMU if necessary.
989 */
990 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
991 {
992 struct unity_map_entry *entry;
993 int ret;
994
995 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
996 if (!iommu_for_unity_map(iommu, entry))
997 continue;
998 ret = dma_ops_unity_map(iommu->default_dom, entry);
999 if (ret)
1000 return ret;
1001 }
1002
1003 return 0;
1004 }
1005
1006 /*
1007 * Inits the unity mappings required for a specific device
1008 */
1009 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
1010 u16 devid)
1011 {
1012 struct unity_map_entry *e;
1013 int ret;
1014
1015 list_for_each_entry(e, &amd_iommu_unity_map, list) {
1016 if (!(devid >= e->devid_start && devid <= e->devid_end))
1017 continue;
1018 ret = dma_ops_unity_map(dma_dom, e);
1019 if (ret)
1020 return ret;
1021 }
1022
1023 return 0;
1024 }
1025
1026 /****************************************************************************
1027 *
1028 * The next functions belong to the address allocator for the dma_ops
1029 * interface functions. They work like the allocators in the other IOMMU
1030 * drivers. Its basically a bitmap which marks the allocated pages in
1031 * the aperture. Maybe it could be enhanced in the future to a more
1032 * efficient allocator.
1033 *
1034 ****************************************************************************/
1035
1036 /*
1037 * The address allocator core functions.
1038 *
1039 * called with domain->lock held
1040 */
1041
1042 /*
1043 * Used to reserve address ranges in the aperture (e.g. for exclusion
1044 * ranges.
1045 */
1046 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1047 unsigned long start_page,
1048 unsigned int pages)
1049 {
1050 unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
1051
1052 if (start_page + pages > last_page)
1053 pages = last_page - start_page;
1054
1055 for (i = start_page; i < start_page + pages; ++i) {
1056 int index = i / APERTURE_RANGE_PAGES;
1057 int page = i % APERTURE_RANGE_PAGES;
1058 __set_bit(page, dom->aperture[index]->bitmap);
1059 }
1060 }
1061
1062 /*
1063 * This function is used to add a new aperture range to an existing
1064 * aperture in case of dma_ops domain allocation or address allocation
1065 * failure.
1066 */
1067 static int alloc_new_range(struct dma_ops_domain *dma_dom,
1068 bool populate, gfp_t gfp)
1069 {
1070 int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1071 struct amd_iommu *iommu;
1072 unsigned long i;
1073
1074 #ifdef CONFIG_IOMMU_STRESS
1075 populate = false;
1076 #endif
1077
1078 if (index >= APERTURE_MAX_RANGES)
1079 return -ENOMEM;
1080
1081 dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
1082 if (!dma_dom->aperture[index])
1083 return -ENOMEM;
1084
1085 dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
1086 if (!dma_dom->aperture[index]->bitmap)
1087 goto out_free;
1088
1089 dma_dom->aperture[index]->offset = dma_dom->aperture_size;
1090
1091 if (populate) {
1092 unsigned long address = dma_dom->aperture_size;
1093 int i, num_ptes = APERTURE_RANGE_PAGES / 512;
1094 u64 *pte, *pte_page;
1095
1096 for (i = 0; i < num_ptes; ++i) {
1097 pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1098 &pte_page, gfp);
1099 if (!pte)
1100 goto out_free;
1101
1102 dma_dom->aperture[index]->pte_pages[i] = pte_page;
1103
1104 address += APERTURE_RANGE_SIZE / 64;
1105 }
1106 }
1107
1108 dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1109
1110 /* Initialize the exclusion range if necessary */
1111 for_each_iommu(iommu) {
1112 if (iommu->exclusion_start &&
1113 iommu->exclusion_start >= dma_dom->aperture[index]->offset
1114 && iommu->exclusion_start < dma_dom->aperture_size) {
1115 unsigned long startpage;
1116 int pages = iommu_num_pages(iommu->exclusion_start,
1117 iommu->exclusion_length,
1118 PAGE_SIZE);
1119 startpage = iommu->exclusion_start >> PAGE_SHIFT;
1120 dma_ops_reserve_addresses(dma_dom, startpage, pages);
1121 }
1122 }
1123
1124 /*
1125 * Check for areas already mapped as present in the new aperture
1126 * range and mark those pages as reserved in the allocator. Such
1127 * mappings may already exist as a result of requested unity
1128 * mappings for devices.
1129 */
1130 for (i = dma_dom->aperture[index]->offset;
1131 i < dma_dom->aperture_size;
1132 i += PAGE_SIZE) {
1133 u64 *pte = fetch_pte(&dma_dom->domain, i);
1134 if (!pte || !IOMMU_PTE_PRESENT(*pte))
1135 continue;
1136
1137 dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);
1138 }
1139
1140 update_domain(&dma_dom->domain);
1141
1142 return 0;
1143
1144 out_free:
1145 update_domain(&dma_dom->domain);
1146
1147 free_page((unsigned long)dma_dom->aperture[index]->bitmap);
1148
1149 kfree(dma_dom->aperture[index]);
1150 dma_dom->aperture[index] = NULL;
1151
1152 return -ENOMEM;
1153 }
1154
1155 static unsigned long dma_ops_area_alloc(struct device *dev,
1156 struct dma_ops_domain *dom,
1157 unsigned int pages,
1158 unsigned long align_mask,
1159 u64 dma_mask,
1160 unsigned long start)
1161 {
1162 unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1163 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1164 int i = start >> APERTURE_RANGE_SHIFT;
1165 unsigned long boundary_size;
1166 unsigned long address = -1;
1167 unsigned long limit;
1168
1169 next_bit >>= PAGE_SHIFT;
1170
1171 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1172 PAGE_SIZE) >> PAGE_SHIFT;
1173
1174 for (;i < max_index; ++i) {
1175 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1176
1177 if (dom->aperture[i]->offset >= dma_mask)
1178 break;
1179
1180 limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1181 dma_mask >> PAGE_SHIFT);
1182
1183 address = iommu_area_alloc(dom->aperture[i]->bitmap,
1184 limit, next_bit, pages, 0,
1185 boundary_size, align_mask);
1186 if (address != -1) {
1187 address = dom->aperture[i]->offset +
1188 (address << PAGE_SHIFT);
1189 dom->next_address = address + (pages << PAGE_SHIFT);
1190 break;
1191 }
1192
1193 next_bit = 0;
1194 }
1195
1196 return address;
1197 }
1198
1199 static unsigned long dma_ops_alloc_addresses(struct device *dev,
1200 struct dma_ops_domain *dom,
1201 unsigned int pages,
1202 unsigned long align_mask,
1203 u64 dma_mask)
1204 {
1205 unsigned long address;
1206
1207 #ifdef CONFIG_IOMMU_STRESS
1208 dom->next_address = 0;
1209 dom->need_flush = true;
1210 #endif
1211
1212 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1213 dma_mask, dom->next_address);
1214
1215 if (address == -1) {
1216 dom->next_address = 0;
1217 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1218 dma_mask, 0);
1219 dom->need_flush = true;
1220 }
1221
1222 if (unlikely(address == -1))
1223 address = DMA_ERROR_CODE;
1224
1225 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1226
1227 return address;
1228 }
1229
1230 /*
1231 * The address free function.
1232 *
1233 * called with domain->lock held
1234 */
1235 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1236 unsigned long address,
1237 unsigned int pages)
1238 {
1239 unsigned i = address >> APERTURE_RANGE_SHIFT;
1240 struct aperture_range *range = dom->aperture[i];
1241
1242 BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1243
1244 #ifdef CONFIG_IOMMU_STRESS
1245 if (i < 4)
1246 return;
1247 #endif
1248
1249 if (address >= dom->next_address)
1250 dom->need_flush = true;
1251
1252 address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1253
1254 bitmap_clear(range->bitmap, address, pages);
1255
1256 }
1257
1258 /****************************************************************************
1259 *
1260 * The next functions belong to the domain allocation. A domain is
1261 * allocated for every IOMMU as the default domain. If device isolation
1262 * is enabled, every device get its own domain. The most important thing
1263 * about domains is the page table mapping the DMA address space they
1264 * contain.
1265 *
1266 ****************************************************************************/
1267
1268 /*
1269 * This function adds a protection domain to the global protection domain list
1270 */
1271 static void add_domain_to_list(struct protection_domain *domain)
1272 {
1273 unsigned long flags;
1274
1275 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1276 list_add(&domain->list, &amd_iommu_pd_list);
1277 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1278 }
1279
1280 /*
1281 * This function removes a protection domain to the global
1282 * protection domain list
1283 */
1284 static void del_domain_from_list(struct protection_domain *domain)
1285 {
1286 unsigned long flags;
1287
1288 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1289 list_del(&domain->list);
1290 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1291 }
1292
1293 static u16 domain_id_alloc(void)
1294 {
1295 unsigned long flags;
1296 int id;
1297
1298 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1299 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1300 BUG_ON(id == 0);
1301 if (id > 0 && id < MAX_DOMAIN_ID)
1302 __set_bit(id, amd_iommu_pd_alloc_bitmap);
1303 else
1304 id = 0;
1305 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1306
1307 return id;
1308 }
1309
1310 static void domain_id_free(int id)
1311 {
1312 unsigned long flags;
1313
1314 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1315 if (id > 0 && id < MAX_DOMAIN_ID)
1316 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1317 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1318 }
1319
1320 static void free_pagetable(struct protection_domain *domain)
1321 {
1322 int i, j;
1323 u64 *p1, *p2, *p3;
1324
1325 p1 = domain->pt_root;
1326
1327 if (!p1)
1328 return;
1329
1330 for (i = 0; i < 512; ++i) {
1331 if (!IOMMU_PTE_PRESENT(p1[i]))
1332 continue;
1333
1334 p2 = IOMMU_PTE_PAGE(p1[i]);
1335 for (j = 0; j < 512; ++j) {
1336 if (!IOMMU_PTE_PRESENT(p2[j]))
1337 continue;
1338 p3 = IOMMU_PTE_PAGE(p2[j]);
1339 free_page((unsigned long)p3);
1340 }
1341
1342 free_page((unsigned long)p2);
1343 }
1344
1345 free_page((unsigned long)p1);
1346
1347 domain->pt_root = NULL;
1348 }
1349
1350 /*
1351 * Free a domain, only used if something went wrong in the
1352 * allocation path and we need to free an already allocated page table
1353 */
1354 static void dma_ops_domain_free(struct dma_ops_domain *dom)
1355 {
1356 int i;
1357
1358 if (!dom)
1359 return;
1360
1361 del_domain_from_list(&dom->domain);
1362
1363 free_pagetable(&dom->domain);
1364
1365 for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
1366 if (!dom->aperture[i])
1367 continue;
1368 free_page((unsigned long)dom->aperture[i]->bitmap);
1369 kfree(dom->aperture[i]);
1370 }
1371
1372 kfree(dom);
1373 }
1374
1375 /*
1376 * Allocates a new protection domain usable for the dma_ops functions.
1377 * It also initializes the page table and the address allocator data
1378 * structures required for the dma_ops interface
1379 */
1380 static struct dma_ops_domain *dma_ops_domain_alloc(void)
1381 {
1382 struct dma_ops_domain *dma_dom;
1383
1384 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
1385 if (!dma_dom)
1386 return NULL;
1387
1388 spin_lock_init(&dma_dom->domain.lock);
1389
1390 dma_dom->domain.id = domain_id_alloc();
1391 if (dma_dom->domain.id == 0)
1392 goto free_dma_dom;
1393 INIT_LIST_HEAD(&dma_dom->domain.dev_list);
1394 dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
1395 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1396 dma_dom->domain.flags = PD_DMA_OPS_MASK;
1397 dma_dom->domain.priv = dma_dom;
1398 if (!dma_dom->domain.pt_root)
1399 goto free_dma_dom;
1400
1401 dma_dom->need_flush = false;
1402 dma_dom->target_dev = 0xffff;
1403
1404 add_domain_to_list(&dma_dom->domain);
1405
1406 if (alloc_new_range(dma_dom, true, GFP_KERNEL))
1407 goto free_dma_dom;
1408
1409 /*
1410 * mark the first page as allocated so we never return 0 as
1411 * a valid dma-address. So we can use 0 as error value
1412 */
1413 dma_dom->aperture[0]->bitmap[0] = 1;
1414 dma_dom->next_address = 0;
1415
1416
1417 return dma_dom;
1418
1419 free_dma_dom:
1420 dma_ops_domain_free(dma_dom);
1421
1422 return NULL;
1423 }
1424
1425 /*
1426 * little helper function to check whether a given protection domain is a
1427 * dma_ops domain
1428 */
1429 static bool dma_ops_domain(struct protection_domain *domain)
1430 {
1431 return domain->flags & PD_DMA_OPS_MASK;
1432 }
1433
1434 static void set_dte_entry(u16 devid, struct protection_domain *domain)
1435 {
1436 u64 pte_root = virt_to_phys(domain->pt_root);
1437
1438 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
1439 << DEV_ENTRY_MODE_SHIFT;
1440 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1441
1442 amd_iommu_dev_table[devid].data[2] = domain->id;
1443 amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
1444 amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
1445 }
1446
1447 static void clear_dte_entry(u16 devid)
1448 {
1449 /* remove entry from the device table seen by the hardware */
1450 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
1451 amd_iommu_dev_table[devid].data[1] = 0;
1452 amd_iommu_dev_table[devid].data[2] = 0;
1453
1454 amd_iommu_apply_erratum_63(devid);
1455 }
1456
1457 static void do_attach(struct device *dev, struct protection_domain *domain)
1458 {
1459 struct iommu_dev_data *dev_data;
1460 struct amd_iommu *iommu;
1461 u16 devid;
1462
1463 devid = get_device_id(dev);
1464 iommu = amd_iommu_rlookup_table[devid];
1465 dev_data = get_dev_data(dev);
1466
1467 /* Update data structures */
1468 dev_data->domain = domain;
1469 list_add(&dev_data->list, &domain->dev_list);
1470 set_dte_entry(devid, domain);
1471
1472 /* Do reference counting */
1473 domain->dev_iommu[iommu->index] += 1;
1474 domain->dev_cnt += 1;
1475
1476 /* Flush the DTE entry */
1477 iommu_flush_device(dev);
1478 }
1479
1480 static void do_detach(struct device *dev)
1481 {
1482 struct iommu_dev_data *dev_data;
1483 struct amd_iommu *iommu;
1484 u16 devid;
1485
1486 devid = get_device_id(dev);
1487 iommu = amd_iommu_rlookup_table[devid];
1488 dev_data = get_dev_data(dev);
1489
1490 /* decrease reference counters */
1491 dev_data->domain->dev_iommu[iommu->index] -= 1;
1492 dev_data->domain->dev_cnt -= 1;
1493
1494 /* Update data structures */
1495 dev_data->domain = NULL;
1496 list_del(&dev_data->list);
1497 clear_dte_entry(devid);
1498
1499 /* Flush the DTE entry */
1500 iommu_flush_device(dev);
1501 }
1502
1503 /*
1504 * If a device is not yet associated with a domain, this function does
1505 * assigns it visible for the hardware
1506 */
1507 static int __attach_device(struct device *dev,
1508 struct protection_domain *domain)
1509 {
1510 struct iommu_dev_data *dev_data, *alias_data;
1511 int ret;
1512
1513 dev_data = get_dev_data(dev);
1514 alias_data = get_dev_data(dev_data->alias);
1515
1516 if (!alias_data)
1517 return -EINVAL;
1518
1519 /* lock domain */
1520 spin_lock(&domain->lock);
1521
1522 /* Some sanity checks */
1523 ret = -EBUSY;
1524 if (alias_data->domain != NULL &&
1525 alias_data->domain != domain)
1526 goto out_unlock;
1527
1528 if (dev_data->domain != NULL &&
1529 dev_data->domain != domain)
1530 goto out_unlock;
1531
1532 /* Do real assignment */
1533 if (dev_data->alias != dev) {
1534 alias_data = get_dev_data(dev_data->alias);
1535 if (alias_data->domain == NULL)
1536 do_attach(dev_data->alias, domain);
1537
1538 atomic_inc(&alias_data->bind);
1539 }
1540
1541 if (dev_data->domain == NULL)
1542 do_attach(dev, domain);
1543
1544 atomic_inc(&dev_data->bind);
1545
1546 ret = 0;
1547
1548 out_unlock:
1549
1550 /* ready */
1551 spin_unlock(&domain->lock);
1552
1553 return ret;
1554 }
1555
1556 /*
1557 * If a device is not yet associated with a domain, this function does
1558 * assigns it visible for the hardware
1559 */
1560 static int attach_device(struct device *dev,
1561 struct protection_domain *domain)
1562 {
1563 unsigned long flags;
1564 int ret;
1565
1566 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1567 ret = __attach_device(dev, domain);
1568 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1569
1570 /*
1571 * We might boot into a crash-kernel here. The crashed kernel
1572 * left the caches in the IOMMU dirty. So we have to flush
1573 * here to evict all dirty stuff.
1574 */
1575 iommu_flush_tlb_pde(domain);
1576
1577 return ret;
1578 }
1579
1580 /*
1581 * Removes a device from a protection domain (unlocked)
1582 */
1583 static void __detach_device(struct device *dev)
1584 {
1585 struct iommu_dev_data *dev_data = get_dev_data(dev);
1586 struct iommu_dev_data *alias_data;
1587 struct protection_domain *domain;
1588 unsigned long flags;
1589
1590 BUG_ON(!dev_data->domain);
1591
1592 domain = dev_data->domain;
1593
1594 spin_lock_irqsave(&domain->lock, flags);
1595
1596 if (dev_data->alias != dev) {
1597 alias_data = get_dev_data(dev_data->alias);
1598 if (atomic_dec_and_test(&alias_data->bind))
1599 do_detach(dev_data->alias);
1600 }
1601
1602 if (atomic_dec_and_test(&dev_data->bind))
1603 do_detach(dev);
1604
1605 spin_unlock_irqrestore(&domain->lock, flags);
1606
1607 /*
1608 * If we run in passthrough mode the device must be assigned to the
1609 * passthrough domain if it is detached from any other domain.
1610 * Make sure we can deassign from the pt_domain itself.
1611 */
1612 if (iommu_pass_through &&
1613 (dev_data->domain == NULL && domain != pt_domain))
1614 __attach_device(dev, pt_domain);
1615 }
1616
1617 /*
1618 * Removes a device from a protection domain (with devtable_lock held)
1619 */
1620 static void detach_device(struct device *dev)
1621 {
1622 unsigned long flags;
1623
1624 /* lock device table */
1625 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1626 __detach_device(dev);
1627 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1628 }
1629
1630 /*
1631 * Find out the protection domain structure for a given PCI device. This
1632 * will give us the pointer to the page table root for example.
1633 */
1634 static struct protection_domain *domain_for_device(struct device *dev)
1635 {
1636 struct protection_domain *dom;
1637 struct iommu_dev_data *dev_data, *alias_data;
1638 unsigned long flags;
1639 u16 devid, alias;
1640
1641 devid = get_device_id(dev);
1642 alias = amd_iommu_alias_table[devid];
1643 dev_data = get_dev_data(dev);
1644 alias_data = get_dev_data(dev_data->alias);
1645 if (!alias_data)
1646 return NULL;
1647
1648 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
1649 dom = dev_data->domain;
1650 if (dom == NULL &&
1651 alias_data->domain != NULL) {
1652 __attach_device(dev, alias_data->domain);
1653 dom = alias_data->domain;
1654 }
1655
1656 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1657
1658 return dom;
1659 }
1660
1661 static int device_change_notifier(struct notifier_block *nb,
1662 unsigned long action, void *data)
1663 {
1664 struct device *dev = data;
1665 u16 devid;
1666 struct protection_domain *domain;
1667 struct dma_ops_domain *dma_domain;
1668 struct amd_iommu *iommu;
1669 unsigned long flags;
1670
1671 if (!check_device(dev))
1672 return 0;
1673
1674 devid = get_device_id(dev);
1675 iommu = amd_iommu_rlookup_table[devid];
1676
1677 switch (action) {
1678 case BUS_NOTIFY_UNBOUND_DRIVER:
1679
1680 domain = domain_for_device(dev);
1681
1682 if (!domain)
1683 goto out;
1684 if (iommu_pass_through)
1685 break;
1686 detach_device(dev);
1687 break;
1688 case BUS_NOTIFY_ADD_DEVICE:
1689
1690 iommu_init_device(dev);
1691
1692 domain = domain_for_device(dev);
1693
1694 /* allocate a protection domain if a device is added */
1695 dma_domain = find_protection_domain(devid);
1696 if (dma_domain)
1697 goto out;
1698 dma_domain = dma_ops_domain_alloc();
1699 if (!dma_domain)
1700 goto out;
1701 dma_domain->target_dev = devid;
1702
1703 spin_lock_irqsave(&iommu_pd_list_lock, flags);
1704 list_add_tail(&dma_domain->list, &iommu_pd_list);
1705 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
1706
1707 break;
1708 case BUS_NOTIFY_DEL_DEVICE:
1709
1710 iommu_uninit_device(dev);
1711
1712 default:
1713 goto out;
1714 }
1715
1716 iommu_flush_device(dev);
1717 iommu_completion_wait(iommu);
1718
1719 out:
1720 return 0;
1721 }
1722
1723 static struct notifier_block device_nb = {
1724 .notifier_call = device_change_notifier,
1725 };
1726
1727 void amd_iommu_init_notifier(void)
1728 {
1729 bus_register_notifier(&pci_bus_type, &device_nb);
1730 }
1731
1732 /*****************************************************************************
1733 *
1734 * The next functions belong to the dma_ops mapping/unmapping code.
1735 *
1736 *****************************************************************************/
1737
1738 /*
1739 * In the dma_ops path we only have the struct device. This function
1740 * finds the corresponding IOMMU, the protection domain and the
1741 * requestor id for a given device.
1742 * If the device is not yet associated with a domain this is also done
1743 * in this function.
1744 */
1745 static struct protection_domain *get_domain(struct device *dev)
1746 {
1747 struct protection_domain *domain;
1748 struct dma_ops_domain *dma_dom;
1749 u16 devid = get_device_id(dev);
1750
1751 if (!check_device(dev))
1752 return ERR_PTR(-EINVAL);
1753
1754 domain = domain_for_device(dev);
1755 if (domain != NULL && !dma_ops_domain(domain))
1756 return ERR_PTR(-EBUSY);
1757
1758 if (domain != NULL)
1759 return domain;
1760
1761 /* Device not bount yet - bind it */
1762 dma_dom = find_protection_domain(devid);
1763 if (!dma_dom)
1764 dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
1765 attach_device(dev, &dma_dom->domain);
1766 DUMP_printk("Using protection domain %d for device %s\n",
1767 dma_dom->domain.id, dev_name(dev));
1768
1769 return &dma_dom->domain;
1770 }
1771
1772 static void update_device_table(struct protection_domain *domain)
1773 {
1774 struct iommu_dev_data *dev_data;
1775
1776 list_for_each_entry(dev_data, &domain->dev_list, list) {
1777 u16 devid = get_device_id(dev_data->dev);
1778 set_dte_entry(devid, domain);
1779 }
1780 }
1781
1782 static void update_domain(struct protection_domain *domain)
1783 {
1784 if (!domain->updated)
1785 return;
1786
1787 update_device_table(domain);
1788 iommu_flush_domain_devices(domain);
1789 iommu_flush_tlb_pde(domain);
1790
1791 domain->updated = false;
1792 }
1793
1794 /*
1795 * This function fetches the PTE for a given address in the aperture
1796 */
1797 static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
1798 unsigned long address)
1799 {
1800 struct aperture_range *aperture;
1801 u64 *pte, *pte_page;
1802
1803 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1804 if (!aperture)
1805 return NULL;
1806
1807 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1808 if (!pte) {
1809 pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
1810 GFP_ATOMIC);
1811 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
1812 } else
1813 pte += PM_LEVEL_INDEX(0, address);
1814
1815 update_domain(&dom->domain);
1816
1817 return pte;
1818 }
1819
1820 /*
1821 * This is the generic map function. It maps one 4kb page at paddr to
1822 * the given address in the DMA address space for the domain.
1823 */
1824 static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
1825 unsigned long address,
1826 phys_addr_t paddr,
1827 int direction)
1828 {
1829 u64 *pte, __pte;
1830
1831 WARN_ON(address > dom->aperture_size);
1832
1833 paddr &= PAGE_MASK;
1834
1835 pte = dma_ops_get_pte(dom, address);
1836 if (!pte)
1837 return DMA_ERROR_CODE;
1838
1839 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
1840
1841 if (direction == DMA_TO_DEVICE)
1842 __pte |= IOMMU_PTE_IR;
1843 else if (direction == DMA_FROM_DEVICE)
1844 __pte |= IOMMU_PTE_IW;
1845 else if (direction == DMA_BIDIRECTIONAL)
1846 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
1847
1848 WARN_ON(*pte);
1849
1850 *pte = __pte;
1851
1852 return (dma_addr_t)address;
1853 }
1854
1855 /*
1856 * The generic unmapping function for on page in the DMA address space.
1857 */
1858 static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
1859 unsigned long address)
1860 {
1861 struct aperture_range *aperture;
1862 u64 *pte;
1863
1864 if (address >= dom->aperture_size)
1865 return;
1866
1867 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1868 if (!aperture)
1869 return;
1870
1871 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1872 if (!pte)
1873 return;
1874
1875 pte += PM_LEVEL_INDEX(0, address);
1876
1877 WARN_ON(!*pte);
1878
1879 *pte = 0ULL;
1880 }
1881
1882 /*
1883 * This function contains common code for mapping of a physically
1884 * contiguous memory region into DMA address space. It is used by all
1885 * mapping functions provided with this IOMMU driver.
1886 * Must be called with the domain lock held.
1887 */
1888 static dma_addr_t __map_single(struct device *dev,
1889 struct dma_ops_domain *dma_dom,
1890 phys_addr_t paddr,
1891 size_t size,
1892 int dir,
1893 bool align,
1894 u64 dma_mask)
1895 {
1896 dma_addr_t offset = paddr & ~PAGE_MASK;
1897 dma_addr_t address, start, ret;
1898 unsigned int pages;
1899 unsigned long align_mask = 0;
1900 int i;
1901
1902 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
1903 paddr &= PAGE_MASK;
1904
1905 INC_STATS_COUNTER(total_map_requests);
1906
1907 if (pages > 1)
1908 INC_STATS_COUNTER(cross_page);
1909
1910 if (align)
1911 align_mask = (1UL << get_order(size)) - 1;
1912
1913 retry:
1914 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
1915 dma_mask);
1916 if (unlikely(address == DMA_ERROR_CODE)) {
1917 /*
1918 * setting next_address here will let the address
1919 * allocator only scan the new allocated range in the
1920 * first run. This is a small optimization.
1921 */
1922 dma_dom->next_address = dma_dom->aperture_size;
1923
1924 if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
1925 goto out;
1926
1927 /*
1928 * aperture was successfully enlarged by 128 MB, try
1929 * allocation again
1930 */
1931 goto retry;
1932 }
1933
1934 start = address;
1935 for (i = 0; i < pages; ++i) {
1936 ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
1937 if (ret == DMA_ERROR_CODE)
1938 goto out_unmap;
1939
1940 paddr += PAGE_SIZE;
1941 start += PAGE_SIZE;
1942 }
1943 address += offset;
1944
1945 ADD_STATS_COUNTER(alloced_io_mem, size);
1946
1947 if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
1948 iommu_flush_tlb(&dma_dom->domain);
1949 dma_dom->need_flush = false;
1950 } else if (unlikely(amd_iommu_np_cache))
1951 iommu_flush_pages(&dma_dom->domain, address, size);
1952
1953 out:
1954 return address;
1955
1956 out_unmap:
1957
1958 for (--i; i >= 0; --i) {
1959 start -= PAGE_SIZE;
1960 dma_ops_domain_unmap(dma_dom, start);
1961 }
1962
1963 dma_ops_free_addresses(dma_dom, address, pages);
1964
1965 return DMA_ERROR_CODE;
1966 }
1967
1968 /*
1969 * Does the reverse of the __map_single function. Must be called with
1970 * the domain lock held too
1971 */
1972 static void __unmap_single(struct dma_ops_domain *dma_dom,
1973 dma_addr_t dma_addr,
1974 size_t size,
1975 int dir)
1976 {
1977 dma_addr_t flush_addr;
1978 dma_addr_t i, start;
1979 unsigned int pages;
1980
1981 if ((dma_addr == DMA_ERROR_CODE) ||
1982 (dma_addr + size > dma_dom->aperture_size))
1983 return;
1984
1985 flush_addr = dma_addr;
1986 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
1987 dma_addr &= PAGE_MASK;
1988 start = dma_addr;
1989
1990 for (i = 0; i < pages; ++i) {
1991 dma_ops_domain_unmap(dma_dom, start);
1992 start += PAGE_SIZE;
1993 }
1994
1995 SUB_STATS_COUNTER(alloced_io_mem, size);
1996
1997 dma_ops_free_addresses(dma_dom, dma_addr, pages);
1998
1999 if (amd_iommu_unmap_flush || dma_dom->need_flush) {
2000 iommu_flush_pages(&dma_dom->domain, flush_addr, size);
2001 dma_dom->need_flush = false;
2002 }
2003 }
2004
2005 /*
2006 * The exported map_single function for dma_ops.
2007 */
2008 static dma_addr_t map_page(struct device *dev, struct page *page,
2009 unsigned long offset, size_t size,
2010 enum dma_data_direction dir,
2011 struct dma_attrs *attrs)
2012 {
2013 unsigned long flags;
2014 struct protection_domain *domain;
2015 dma_addr_t addr;
2016 u64 dma_mask;
2017 phys_addr_t paddr = page_to_phys(page) + offset;
2018
2019 INC_STATS_COUNTER(cnt_map_single);
2020
2021 domain = get_domain(dev);
2022 if (PTR_ERR(domain) == -EINVAL)
2023 return (dma_addr_t)paddr;
2024 else if (IS_ERR(domain))
2025 return DMA_ERROR_CODE;
2026
2027 dma_mask = *dev->dma_mask;
2028
2029 spin_lock_irqsave(&domain->lock, flags);
2030
2031 addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2032 dma_mask);
2033 if (addr == DMA_ERROR_CODE)
2034 goto out;
2035
2036 iommu_flush_complete(domain);
2037
2038 out:
2039 spin_unlock_irqrestore(&domain->lock, flags);
2040
2041 return addr;
2042 }
2043
2044 /*
2045 * The exported unmap_single function for dma_ops.
2046 */
2047 static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2048 enum dma_data_direction dir, struct dma_attrs *attrs)
2049 {
2050 unsigned long flags;
2051 struct protection_domain *domain;
2052
2053 INC_STATS_COUNTER(cnt_unmap_single);
2054
2055 domain = get_domain(dev);
2056 if (IS_ERR(domain))
2057 return;
2058
2059 spin_lock_irqsave(&domain->lock, flags);
2060
2061 __unmap_single(domain->priv, dma_addr, size, dir);
2062
2063 iommu_flush_complete(domain);
2064
2065 spin_unlock_irqrestore(&domain->lock, flags);
2066 }
2067
2068 /*
2069 * This is a special map_sg function which is used if we should map a
2070 * device which is not handled by an AMD IOMMU in the system.
2071 */
2072 static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
2073 int nelems, int dir)
2074 {
2075 struct scatterlist *s;
2076 int i;
2077
2078 for_each_sg(sglist, s, nelems, i) {
2079 s->dma_address = (dma_addr_t)sg_phys(s);
2080 s->dma_length = s->length;
2081 }
2082
2083 return nelems;
2084 }
2085
2086 /*
2087 * The exported map_sg function for dma_ops (handles scatter-gather
2088 * lists).
2089 */
2090 static int map_sg(struct device *dev, struct scatterlist *sglist,
2091 int nelems, enum dma_data_direction dir,
2092 struct dma_attrs *attrs)
2093 {
2094 unsigned long flags;
2095 struct protection_domain *domain;
2096 int i;
2097 struct scatterlist *s;
2098 phys_addr_t paddr;
2099 int mapped_elems = 0;
2100 u64 dma_mask;
2101
2102 INC_STATS_COUNTER(cnt_map_sg);
2103
2104 domain = get_domain(dev);
2105 if (PTR_ERR(domain) == -EINVAL)
2106 return map_sg_no_iommu(dev, sglist, nelems, dir);
2107 else if (IS_ERR(domain))
2108 return 0;
2109
2110 dma_mask = *dev->dma_mask;
2111
2112 spin_lock_irqsave(&domain->lock, flags);
2113
2114 for_each_sg(sglist, s, nelems, i) {
2115 paddr = sg_phys(s);
2116
2117 s->dma_address = __map_single(dev, domain->priv,
2118 paddr, s->length, dir, false,
2119 dma_mask);
2120
2121 if (s->dma_address) {
2122 s->dma_length = s->length;
2123 mapped_elems++;
2124 } else
2125 goto unmap;
2126 }
2127
2128 iommu_flush_complete(domain);
2129
2130 out:
2131 spin_unlock_irqrestore(&domain->lock, flags);
2132
2133 return mapped_elems;
2134 unmap:
2135 for_each_sg(sglist, s, mapped_elems, i) {
2136 if (s->dma_address)
2137 __unmap_single(domain->priv, s->dma_address,
2138 s->dma_length, dir);
2139 s->dma_address = s->dma_length = 0;
2140 }
2141
2142 mapped_elems = 0;
2143
2144 goto out;
2145 }
2146
2147 /*
2148 * The exported map_sg function for dma_ops (handles scatter-gather
2149 * lists).
2150 */
2151 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2152 int nelems, enum dma_data_direction dir,
2153 struct dma_attrs *attrs)
2154 {
2155 unsigned long flags;
2156 struct protection_domain *domain;
2157 struct scatterlist *s;
2158 int i;
2159
2160 INC_STATS_COUNTER(cnt_unmap_sg);
2161
2162 domain = get_domain(dev);
2163 if (IS_ERR(domain))
2164 return;
2165
2166 spin_lock_irqsave(&domain->lock, flags);
2167
2168 for_each_sg(sglist, s, nelems, i) {
2169 __unmap_single(domain->priv, s->dma_address,
2170 s->dma_length, dir);
2171 s->dma_address = s->dma_length = 0;
2172 }
2173
2174 iommu_flush_complete(domain);
2175
2176 spin_unlock_irqrestore(&domain->lock, flags);
2177 }
2178
2179 /*
2180 * The exported alloc_coherent function for dma_ops.
2181 */
2182 static void *alloc_coherent(struct device *dev, size_t size,
2183 dma_addr_t *dma_addr, gfp_t flag)
2184 {
2185 unsigned long flags;
2186 void *virt_addr;
2187 struct protection_domain *domain;
2188 phys_addr_t paddr;
2189 u64 dma_mask = dev->coherent_dma_mask;
2190
2191 INC_STATS_COUNTER(cnt_alloc_coherent);
2192
2193 domain = get_domain(dev);
2194 if (PTR_ERR(domain) == -EINVAL) {
2195 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2196 *dma_addr = __pa(virt_addr);
2197 return virt_addr;
2198 } else if (IS_ERR(domain))
2199 return NULL;
2200
2201 dma_mask = dev->coherent_dma_mask;
2202 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2203 flag |= __GFP_ZERO;
2204
2205 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2206 if (!virt_addr)
2207 return NULL;
2208
2209 paddr = virt_to_phys(virt_addr);
2210
2211 if (!dma_mask)
2212 dma_mask = *dev->dma_mask;
2213
2214 spin_lock_irqsave(&domain->lock, flags);
2215
2216 *dma_addr = __map_single(dev, domain->priv, paddr,
2217 size, DMA_BIDIRECTIONAL, true, dma_mask);
2218
2219 if (*dma_addr == DMA_ERROR_CODE) {
2220 spin_unlock_irqrestore(&domain->lock, flags);
2221 goto out_free;
2222 }
2223
2224 iommu_flush_complete(domain);
2225
2226 spin_unlock_irqrestore(&domain->lock, flags);
2227
2228 return virt_addr;
2229
2230 out_free:
2231
2232 free_pages((unsigned long)virt_addr, get_order(size));
2233
2234 return NULL;
2235 }
2236
2237 /*
2238 * The exported free_coherent function for dma_ops.
2239 */
2240 static void free_coherent(struct device *dev, size_t size,
2241 void *virt_addr, dma_addr_t dma_addr)
2242 {
2243 unsigned long flags;
2244 struct protection_domain *domain;
2245
2246 INC_STATS_COUNTER(cnt_free_coherent);
2247
2248 domain = get_domain(dev);
2249 if (IS_ERR(domain))
2250 goto free_mem;
2251
2252 spin_lock_irqsave(&domain->lock, flags);
2253
2254 __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2255
2256 iommu_flush_complete(domain);
2257
2258 spin_unlock_irqrestore(&domain->lock, flags);
2259
2260 free_mem:
2261 free_pages((unsigned long)virt_addr, get_order(size));
2262 }
2263
2264 /*
2265 * This function is called by the DMA layer to find out if we can handle a
2266 * particular device. It is part of the dma_ops.
2267 */
2268 static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2269 {
2270 return check_device(dev);
2271 }
2272
2273 /*
2274 * The function for pre-allocating protection domains.
2275 *
2276 * If the driver core informs the DMA layer if a driver grabs a device
2277 * we don't need to preallocate the protection domains anymore.
2278 * For now we have to.
2279 */
2280 static void prealloc_protection_domains(void)
2281 {
2282 struct pci_dev *dev = NULL;
2283 struct dma_ops_domain *dma_dom;
2284 u16 devid;
2285
2286 for_each_pci_dev(dev) {
2287
2288 /* Do we handle this device? */
2289 if (!check_device(&dev->dev))
2290 continue;
2291
2292 /* Is there already any domain for it? */
2293 if (domain_for_device(&dev->dev))
2294 continue;
2295
2296 devid = get_device_id(&dev->dev);
2297
2298 dma_dom = dma_ops_domain_alloc();
2299 if (!dma_dom)
2300 continue;
2301 init_unity_mappings_for_device(dma_dom, devid);
2302 dma_dom->target_dev = devid;
2303
2304 attach_device(&dev->dev, &dma_dom->domain);
2305
2306 list_add_tail(&dma_dom->list, &iommu_pd_list);
2307 }
2308 }
2309
2310 static struct dma_map_ops amd_iommu_dma_ops = {
2311 .alloc_coherent = alloc_coherent,
2312 .free_coherent = free_coherent,
2313 .map_page = map_page,
2314 .unmap_page = unmap_page,
2315 .map_sg = map_sg,
2316 .unmap_sg = unmap_sg,
2317 .dma_supported = amd_iommu_dma_supported,
2318 };
2319
2320 static unsigned device_dma_ops_init(void)
2321 {
2322 struct pci_dev *pdev = NULL;
2323 unsigned unhandled = 0;
2324
2325 for_each_pci_dev(pdev) {
2326 if (!check_device(&pdev->dev)) {
2327 unhandled += 1;
2328 continue;
2329 }
2330
2331 pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
2332 }
2333
2334 return unhandled;
2335 }
2336
2337 /*
2338 * The function which clues the AMD IOMMU driver into dma_ops.
2339 */
2340
2341 void __init amd_iommu_init_api(void)
2342 {
2343 register_iommu(&amd_iommu_ops);
2344 }
2345
2346 int __init amd_iommu_init_dma_ops(void)
2347 {
2348 struct amd_iommu *iommu;
2349 int ret, unhandled;
2350
2351 /*
2352 * first allocate a default protection domain for every IOMMU we
2353 * found in the system. Devices not assigned to any other
2354 * protection domain will be assigned to the default one.
2355 */
2356 for_each_iommu(iommu) {
2357 iommu->default_dom = dma_ops_domain_alloc();
2358 if (iommu->default_dom == NULL)
2359 return -ENOMEM;
2360 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
2361 ret = iommu_init_unity_mappings(iommu);
2362 if (ret)
2363 goto free_domains;
2364 }
2365
2366 /*
2367 * Pre-allocate the protection domains for each device.
2368 */
2369 prealloc_protection_domains();
2370
2371 iommu_detected = 1;
2372 swiotlb = 0;
2373
2374 /* Make the driver finally visible to the drivers */
2375 unhandled = device_dma_ops_init();
2376 if (unhandled && max_pfn > MAX_DMA32_PFN) {
2377 /* There are unhandled devices - initialize swiotlb for them */
2378 swiotlb = 1;
2379 }
2380
2381 amd_iommu_stats_init();
2382
2383 return 0;
2384
2385 free_domains:
2386
2387 for_each_iommu(iommu) {
2388 if (iommu->default_dom)
2389 dma_ops_domain_free(iommu->default_dom);
2390 }
2391
2392 return ret;
2393 }
2394
2395 /*****************************************************************************
2396 *
2397 * The following functions belong to the exported interface of AMD IOMMU
2398 *
2399 * This interface allows access to lower level functions of the IOMMU
2400 * like protection domain handling and assignement of devices to domains
2401 * which is not possible with the dma_ops interface.
2402 *
2403 *****************************************************************************/
2404
2405 static void cleanup_domain(struct protection_domain *domain)
2406 {
2407 struct iommu_dev_data *dev_data, *next;
2408 unsigned long flags;
2409
2410 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2411
2412 list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {
2413 struct device *dev = dev_data->dev;
2414
2415 __detach_device(dev);
2416 atomic_set(&dev_data->bind, 0);
2417 }
2418
2419 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2420 }
2421
2422 static void protection_domain_free(struct protection_domain *domain)
2423 {
2424 if (!domain)
2425 return;
2426
2427 del_domain_from_list(domain);
2428
2429 if (domain->id)
2430 domain_id_free(domain->id);
2431
2432 kfree(domain);
2433 }
2434
2435 static struct protection_domain *protection_domain_alloc(void)
2436 {
2437 struct protection_domain *domain;
2438
2439 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2440 if (!domain)
2441 return NULL;
2442
2443 spin_lock_init(&domain->lock);
2444 mutex_init(&domain->api_lock);
2445 domain->id = domain_id_alloc();
2446 if (!domain->id)
2447 goto out_err;
2448 INIT_LIST_HEAD(&domain->dev_list);
2449
2450 add_domain_to_list(domain);
2451
2452 return domain;
2453
2454 out_err:
2455 kfree(domain);
2456
2457 return NULL;
2458 }
2459
2460 static int amd_iommu_domain_init(struct iommu_domain *dom)
2461 {
2462 struct protection_domain *domain;
2463
2464 domain = protection_domain_alloc();
2465 if (!domain)
2466 goto out_free;
2467
2468 domain->mode = PAGE_MODE_3_LEVEL;
2469 domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2470 if (!domain->pt_root)
2471 goto out_free;
2472
2473 dom->priv = domain;
2474
2475 return 0;
2476
2477 out_free:
2478 protection_domain_free(domain);
2479
2480 return -ENOMEM;
2481 }
2482
2483 static void amd_iommu_domain_destroy(struct iommu_domain *dom)
2484 {
2485 struct protection_domain *domain = dom->priv;
2486
2487 if (!domain)
2488 return;
2489
2490 if (domain->dev_cnt > 0)
2491 cleanup_domain(domain);
2492
2493 BUG_ON(domain->dev_cnt != 0);
2494
2495 free_pagetable(domain);
2496
2497 protection_domain_free(domain);
2498
2499 dom->priv = NULL;
2500 }
2501
2502 static void amd_iommu_detach_device(struct iommu_domain *dom,
2503 struct device *dev)
2504 {
2505 struct iommu_dev_data *dev_data = dev->archdata.iommu;
2506 struct amd_iommu *iommu;
2507 u16 devid;
2508
2509 if (!check_device(dev))
2510 return;
2511
2512 devid = get_device_id(dev);
2513
2514 if (dev_data->domain != NULL)
2515 detach_device(dev);
2516
2517 iommu = amd_iommu_rlookup_table[devid];
2518 if (!iommu)
2519 return;
2520
2521 iommu_flush_device(dev);
2522 iommu_completion_wait(iommu);
2523 }
2524
2525 static int amd_iommu_attach_device(struct iommu_domain *dom,
2526 struct device *dev)
2527 {
2528 struct protection_domain *domain = dom->priv;
2529 struct iommu_dev_data *dev_data;
2530 struct amd_iommu *iommu;
2531 int ret;
2532 u16 devid;
2533
2534 if (!check_device(dev))
2535 return -EINVAL;
2536
2537 dev_data = dev->archdata.iommu;
2538
2539 devid = get_device_id(dev);
2540
2541 iommu = amd_iommu_rlookup_table[devid];
2542 if (!iommu)
2543 return -EINVAL;
2544
2545 if (dev_data->domain)
2546 detach_device(dev);
2547
2548 ret = attach_device(dev, domain);
2549
2550 iommu_completion_wait(iommu);
2551
2552 return ret;
2553 }
2554
2555 static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
2556 phys_addr_t paddr, int gfp_order, int iommu_prot)
2557 {
2558 unsigned long page_size = 0x1000UL << gfp_order;
2559 struct protection_domain *domain = dom->priv;
2560 int prot = 0;
2561 int ret;
2562
2563 if (iommu_prot & IOMMU_READ)
2564 prot |= IOMMU_PROT_IR;
2565 if (iommu_prot & IOMMU_WRITE)
2566 prot |= IOMMU_PROT_IW;
2567
2568 mutex_lock(&domain->api_lock);
2569 ret = iommu_map_page(domain, iova, paddr, prot, page_size);
2570 mutex_unlock(&domain->api_lock);
2571
2572 return ret;
2573 }
2574
2575 static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
2576 int gfp_order)
2577 {
2578 struct protection_domain *domain = dom->priv;
2579 unsigned long page_size, unmap_size;
2580
2581 page_size = 0x1000UL << gfp_order;
2582
2583 mutex_lock(&domain->api_lock);
2584 unmap_size = iommu_unmap_page(domain, iova, page_size);
2585 mutex_unlock(&domain->api_lock);
2586
2587 iommu_flush_tlb_pde(domain);
2588
2589 return get_order(unmap_size);
2590 }
2591
2592 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2593 unsigned long iova)
2594 {
2595 struct protection_domain *domain = dom->priv;
2596 unsigned long offset_mask;
2597 phys_addr_t paddr;
2598 u64 *pte, __pte;
2599
2600 pte = fetch_pte(domain, iova);
2601
2602 if (!pte || !IOMMU_PTE_PRESENT(*pte))
2603 return 0;
2604
2605 if (PM_PTE_LEVEL(*pte) == 0)
2606 offset_mask = PAGE_SIZE - 1;
2607 else
2608 offset_mask = PTE_PAGE_SIZE(*pte) - 1;
2609
2610 __pte = *pte & PM_ADDR_MASK;
2611 paddr = (__pte & ~offset_mask) | (iova & offset_mask);
2612
2613 return paddr;
2614 }
2615
2616 static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
2617 unsigned long cap)
2618 {
2619 switch (cap) {
2620 case IOMMU_CAP_CACHE_COHERENCY:
2621 return 1;
2622 }
2623
2624 return 0;
2625 }
2626
2627 static struct iommu_ops amd_iommu_ops = {
2628 .domain_init = amd_iommu_domain_init,
2629 .domain_destroy = amd_iommu_domain_destroy,
2630 .attach_dev = amd_iommu_attach_device,
2631 .detach_dev = amd_iommu_detach_device,
2632 .map = amd_iommu_map,
2633 .unmap = amd_iommu_unmap,
2634 .iova_to_phys = amd_iommu_iova_to_phys,
2635 .domain_has_cap = amd_iommu_domain_has_cap,
2636 };
2637
2638 /*****************************************************************************
2639 *
2640 * The next functions do a basic initialization of IOMMU for pass through
2641 * mode
2642 *
2643 * In passthrough mode the IOMMU is initialized and enabled but not used for
2644 * DMA-API translation.
2645 *
2646 *****************************************************************************/
2647
2648 int __init amd_iommu_init_passthrough(void)
2649 {
2650 struct amd_iommu *iommu;
2651 struct pci_dev *dev = NULL;
2652 u16 devid;
2653
2654 /* allocate passthrough domain */
2655 pt_domain = protection_domain_alloc();
2656 if (!pt_domain)
2657 return -ENOMEM;
2658
2659 pt_domain->mode |= PAGE_MODE_NONE;
2660
2661 for_each_pci_dev(dev) {
2662 if (!check_device(&dev->dev))
2663 continue;
2664
2665 devid = get_device_id(&dev->dev);
2666
2667 iommu = amd_iommu_rlookup_table[devid];
2668 if (!iommu)
2669 continue;
2670
2671 attach_device(&dev->dev, pt_domain);
2672 }
2673
2674 pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
2675
2676 return 0;
2677 }
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree