?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/amd-iommu: Cleanup attach/detach_device code
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / arch / x86 / kernel / amd_iommu.c
blobe3363fd5eef5b0dd436f49cecffbec760f05e57a
1 /*
2 * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/pci.h>
21 #include <linux/gfp.h>
22 #include <linux/bitops.h>
23 #include <linux/debugfs.h>
24 #include <linux/scatterlist.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/iommu-helper.h>
27 #include <linux/iommu.h>
28 #include <asm/proto.h>
29 #include <asm/iommu.h>
30 #include <asm/gart.h>
31 #include <asm/amd_iommu_proto.h>
32 #include <asm/amd_iommu_types.h>
33 #include <asm/amd_iommu.h>
34
35 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
36
37 #define EXIT_LOOP_COUNT 10000000
38
39 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
40
41 /* A list of preallocated protection domains */
42 static LIST_HEAD(iommu_pd_list);
43 static DEFINE_SPINLOCK(iommu_pd_list_lock);
44
45 /*
46 * Domain for untranslated devices - only allocated
47 * if iommu=pt passed on kernel cmd line.
48 */
49 static struct protection_domain *pt_domain;
50
51 static struct iommu_ops amd_iommu_ops;
52
53 /*
54 * general struct to manage commands send to an IOMMU
55 */
56 struct iommu_cmd {
57 u32 data[4];
58 };
59
60 static void reset_iommu_command_buffer(struct amd_iommu *iommu);
61 static void update_domain(struct protection_domain *domain);
62
63 /****************************************************************************
64 *
65 * Helper functions
66 *
67 ****************************************************************************/
68
69 static inline u16 get_device_id(struct device *dev)
70 {
71 struct pci_dev *pdev = to_pci_dev(dev);
72
73 return calc_devid(pdev->bus->number, pdev->devfn);
74 }
75
76 static struct iommu_dev_data *get_dev_data(struct device *dev)
77 {
78 return dev->archdata.iommu;
79 }
80
81 /*
82 * In this function the list of preallocated protection domains is traversed to
83 * find the domain for a specific device
84 */
85 static struct dma_ops_domain *find_protection_domain(u16 devid)
86 {
87 struct dma_ops_domain *entry, *ret = NULL;
88 unsigned long flags;
89 u16 alias = amd_iommu_alias_table[devid];
90
91 if (list_empty(&iommu_pd_list))
92 return NULL;
93
94 spin_lock_irqsave(&iommu_pd_list_lock, flags);
95
96 list_for_each_entry(entry, &iommu_pd_list, list) {
97 if (entry->target_dev == devid ||
98 entry->target_dev == alias) {
99 ret = entry;
100 break;
101 }
102 }
103
104 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
105
106 return ret;
107 }
108
109 /*
110 * This function checks if the driver got a valid device from the caller to
111 * avoid dereferencing invalid pointers.
112 */
113 static bool check_device(struct device *dev)
114 {
115 u16 devid;
116
117 if (!dev || !dev->dma_mask)
118 return false;
119
120 /* No device or no PCI device */
121 if (!dev || dev->bus != &pci_bus_type)
122 return false;
123
124 devid = get_device_id(dev);
125
126 /* Out of our scope? */
127 if (devid > amd_iommu_last_bdf)
128 return false;
129
130 if (amd_iommu_rlookup_table[devid] == NULL)
131 return false;
132
133 return true;
134 }
135
136 static int iommu_init_device(struct device *dev)
137 {
138 struct iommu_dev_data *dev_data;
139 struct pci_dev *pdev;
140 u16 devid, alias;
141
142 if (dev->archdata.iommu)
143 return 0;
144
145 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
146 if (!dev_data)
147 return -ENOMEM;
148
149 devid = get_device_id(dev);
150 alias = amd_iommu_alias_table[devid];
151 pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff);
152 if (pdev)
153 dev_data->alias = &pdev->dev;
154
155 atomic_set(&dev_data->bind, 0);
156
157 dev->archdata.iommu = dev_data;
158
159
160 return 0;
161 }
162
163 static void iommu_uninit_device(struct device *dev)
164 {
165 kfree(dev->archdata.iommu);
166 }
167 #ifdef CONFIG_AMD_IOMMU_STATS
168
169 /*
170 * Initialization code for statistics collection
171 */
172
173 DECLARE_STATS_COUNTER(compl_wait);
174 DECLARE_STATS_COUNTER(cnt_map_single);
175 DECLARE_STATS_COUNTER(cnt_unmap_single);
176 DECLARE_STATS_COUNTER(cnt_map_sg);
177 DECLARE_STATS_COUNTER(cnt_unmap_sg);
178 DECLARE_STATS_COUNTER(cnt_alloc_coherent);
179 DECLARE_STATS_COUNTER(cnt_free_coherent);
180 DECLARE_STATS_COUNTER(cross_page);
181 DECLARE_STATS_COUNTER(domain_flush_single);
182 DECLARE_STATS_COUNTER(domain_flush_all);
183 DECLARE_STATS_COUNTER(alloced_io_mem);
184 DECLARE_STATS_COUNTER(total_map_requests);
185
186 static struct dentry *stats_dir;
187 static struct dentry *de_fflush;
188
189 static void amd_iommu_stats_add(struct __iommu_counter *cnt)
190 {
191 if (stats_dir == NULL)
192 return;
193
194 cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
195 &cnt->value);
196 }
197
198 static void amd_iommu_stats_init(void)
199 {
200 stats_dir = debugfs_create_dir("amd-iommu", NULL);
201 if (stats_dir == NULL)
202 return;
203
204 de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
205 (u32 *)&amd_iommu_unmap_flush);
206
207 amd_iommu_stats_add(&compl_wait);
208 amd_iommu_stats_add(&cnt_map_single);
209 amd_iommu_stats_add(&cnt_unmap_single);
210 amd_iommu_stats_add(&cnt_map_sg);
211 amd_iommu_stats_add(&cnt_unmap_sg);
212 amd_iommu_stats_add(&cnt_alloc_coherent);
213 amd_iommu_stats_add(&cnt_free_coherent);
214 amd_iommu_stats_add(&cross_page);
215 amd_iommu_stats_add(&domain_flush_single);
216 amd_iommu_stats_add(&domain_flush_all);
217 amd_iommu_stats_add(&alloced_io_mem);
218 amd_iommu_stats_add(&total_map_requests);
219 }
220
221 #endif
222
223 /****************************************************************************
224 *
225 * Interrupt handling functions
226 *
227 ****************************************************************************/
228
229 static void dump_dte_entry(u16 devid)
230 {
231 int i;
232
233 for (i = 0; i < 8; ++i)
234 pr_err("AMD-Vi: DTE[%d]: %08x\n", i,
235 amd_iommu_dev_table[devid].data[i]);
236 }
237
238 static void dump_command(unsigned long phys_addr)
239 {
240 struct iommu_cmd *cmd = phys_to_virt(phys_addr);
241 int i;
242
243 for (i = 0; i < 4; ++i)
244 pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
245 }
246
247 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
248 {
249 u32 *event = __evt;
250 int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
251 int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
252 int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
253 int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
254 u64 address = (u64)(((u64)event[3]) << 32) | event[2];
255
256 printk(KERN_ERR "AMD-Vi: Event logged [");
257
258 switch (type) {
259 case EVENT_TYPE_ILL_DEV:
260 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
261 "address=0x%016llx flags=0x%04x]\n",
262 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
263 address, flags);
264 dump_dte_entry(devid);
265 break;
266 case EVENT_TYPE_IO_FAULT:
267 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
268 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
269 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
270 domid, address, flags);
271 break;
272 case EVENT_TYPE_DEV_TAB_ERR:
273 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
274 "address=0x%016llx flags=0x%04x]\n",
275 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
276 address, flags);
277 break;
278 case EVENT_TYPE_PAGE_TAB_ERR:
279 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
280 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
281 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
282 domid, address, flags);
283 break;
284 case EVENT_TYPE_ILL_CMD:
285 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
286 reset_iommu_command_buffer(iommu);
287 dump_command(address);
288 break;
289 case EVENT_TYPE_CMD_HARD_ERR:
290 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
291 "flags=0x%04x]\n", address, flags);
292 break;
293 case EVENT_TYPE_IOTLB_INV_TO:
294 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
295 "address=0x%016llx]\n",
296 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
297 address);
298 break;
299 case EVENT_TYPE_INV_DEV_REQ:
300 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
301 "address=0x%016llx flags=0x%04x]\n",
302 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
303 address, flags);
304 break;
305 default:
306 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
307 }
308 }
309
310 static void iommu_poll_events(struct amd_iommu *iommu)
311 {
312 u32 head, tail;
313 unsigned long flags;
314
315 spin_lock_irqsave(&iommu->lock, flags);
316
317 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
318 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
319
320 while (head != tail) {
321 iommu_print_event(iommu, iommu->evt_buf + head);
322 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
323 }
324
325 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
326
327 spin_unlock_irqrestore(&iommu->lock, flags);
328 }
329
330 irqreturn_t amd_iommu_int_handler(int irq, void *data)
331 {
332 struct amd_iommu *iommu;
333
334 for_each_iommu(iommu)
335 iommu_poll_events(iommu);
336
337 return IRQ_HANDLED;
338 }
339
340 /****************************************************************************
341 *
342 * IOMMU command queuing functions
343 *
344 ****************************************************************************/
345
346 /*
347 * Writes the command to the IOMMUs command buffer and informs the
348 * hardware about the new command. Must be called with iommu->lock held.
349 */
350 static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
351 {
352 u32 tail, head;
353 u8 *target;
354
355 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
356 target = iommu->cmd_buf + tail;
357 memcpy_toio(target, cmd, sizeof(*cmd));
358 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
359 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
360 if (tail == head)
361 return -ENOMEM;
362 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
363
364 return 0;
365 }
366
367 /*
368 * General queuing function for commands. Takes iommu->lock and calls
369 * __iommu_queue_command().
370 */
371 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
372 {
373 unsigned long flags;
374 int ret;
375
376 spin_lock_irqsave(&iommu->lock, flags);
377 ret = __iommu_queue_command(iommu, cmd);
378 if (!ret)
379 iommu->need_sync = true;
380 spin_unlock_irqrestore(&iommu->lock, flags);
381
382 return ret;
383 }
384
385 /*
386 * This function waits until an IOMMU has completed a completion
387 * wait command
388 */
389 static void __iommu_wait_for_completion(struct amd_iommu *iommu)
390 {
391 int ready = 0;
392 unsigned status = 0;
393 unsigned long i = 0;
394
395 INC_STATS_COUNTER(compl_wait);
396
397 while (!ready && (i < EXIT_LOOP_COUNT)) {
398 ++i;
399 /* wait for the bit to become one */
400 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
401 ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
402 }
403
404 /* set bit back to zero */
405 status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
406 writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
407
408 if (unlikely(i == EXIT_LOOP_COUNT)) {
409 spin_unlock(&iommu->lock);
410 reset_iommu_command_buffer(iommu);
411 spin_lock(&iommu->lock);
412 }
413 }
414
415 /*
416 * This function queues a completion wait command into the command
417 * buffer of an IOMMU
418 */
419 static int __iommu_completion_wait(struct amd_iommu *iommu)
420 {
421 struct iommu_cmd cmd;
422
423 memset(&cmd, 0, sizeof(cmd));
424 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
425 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
426
427 return __iommu_queue_command(iommu, &cmd);
428 }
429
430 /*
431 * This function is called whenever we need to ensure that the IOMMU has
432 * completed execution of all commands we sent. It sends a
433 * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
434 * us about that by writing a value to a physical address we pass with
435 * the command.
436 */
437 static int iommu_completion_wait(struct amd_iommu *iommu)
438 {
439 int ret = 0;
440 unsigned long flags;
441
442 spin_lock_irqsave(&iommu->lock, flags);
443
444 if (!iommu->need_sync)
445 goto out;
446
447 ret = __iommu_completion_wait(iommu);
448
449 iommu->need_sync = false;
450
451 if (ret)
452 goto out;
453
454 __iommu_wait_for_completion(iommu);
455
456 out:
457 spin_unlock_irqrestore(&iommu->lock, flags);
458
459 return 0;
460 }
461
462 static void iommu_flush_complete(struct protection_domain *domain)
463 {
464 int i;
465
466 for (i = 0; i < amd_iommus_present; ++i) {
467 if (!domain->dev_iommu[i])
468 continue;
469
470 /*
471 * Devices of this domain are behind this IOMMU
472 * We need to wait for completion of all commands.
473 */
474 iommu_completion_wait(amd_iommus[i]);
475 }
476 }
477
478 /*
479 * Command send function for invalidating a device table entry
480 */
481 static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
482 {
483 struct iommu_cmd cmd;
484 int ret;
485
486 BUG_ON(iommu == NULL);
487
488 memset(&cmd, 0, sizeof(cmd));
489 CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
490 cmd.data[0] = devid;
491
492 ret = iommu_queue_command(iommu, &cmd);
493
494 return ret;
495 }
496
497 static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
498 u16 domid, int pde, int s)
499 {
500 memset(cmd, 0, sizeof(*cmd));
501 address &= PAGE_MASK;
502 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
503 cmd->data[1] |= domid;
504 cmd->data[2] = lower_32_bits(address);
505 cmd->data[3] = upper_32_bits(address);
506 if (s) /* size bit - we flush more than one 4kb page */
507 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
508 if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
509 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
510 }
511
512 /*
513 * Generic command send function for invalidaing TLB entries
514 */
515 static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
516 u64 address, u16 domid, int pde, int s)
517 {
518 struct iommu_cmd cmd;
519 int ret;
520
521 __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
522
523 ret = iommu_queue_command(iommu, &cmd);
524
525 return ret;
526 }
527
528 /*
529 * TLB invalidation function which is called from the mapping functions.
530 * It invalidates a single PTE if the range to flush is within a single
531 * page. Otherwise it flushes the whole TLB of the IOMMU.
532 */
533 static void __iommu_flush_pages(struct protection_domain *domain,
534 u64 address, size_t size, int pde)
535 {
536 int s = 0, i;
537 unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE);
538
539 address &= PAGE_MASK;
540
541 if (pages > 1) {
542 /*
543 * If we have to flush more than one page, flush all
544 * TLB entries for this domain
545 */
546 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
547 s = 1;
548 }
549
550
551 for (i = 0; i < amd_iommus_present; ++i) {
552 if (!domain->dev_iommu[i])
553 continue;
554
555 /*
556 * Devices of this domain are behind this IOMMU
557 * We need a TLB flush
558 */
559 iommu_queue_inv_iommu_pages(amd_iommus[i], address,
560 domain->id, pde, s);
561 }
562
563 return;
564 }
565
566 static void iommu_flush_pages(struct protection_domain *domain,
567 u64 address, size_t size)
568 {
569 __iommu_flush_pages(domain, address, size, 0);
570 }
571
572 /* Flush the whole IO/TLB for a given protection domain */
573 static void iommu_flush_tlb(struct protection_domain *domain)
574 {
575 __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
576 }
577
578 /* Flush the whole IO/TLB for a given protection domain - including PDE */
579 static void iommu_flush_tlb_pde(struct protection_domain *domain)
580 {
581 __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
582 }
583
584 /*
585 * This function flushes all domains that have devices on the given IOMMU
586 */
587 static void flush_all_domains_on_iommu(struct amd_iommu *iommu)
588 {
589 u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
590 struct protection_domain *domain;
591 unsigned long flags;
592
593 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
594
595 list_for_each_entry(domain, &amd_iommu_pd_list, list) {
596 if (domain->dev_iommu[iommu->index] == 0)
597 continue;
598
599 spin_lock(&domain->lock);
600 iommu_queue_inv_iommu_pages(iommu, address, domain->id, 1, 1);
601 iommu_flush_complete(domain);
602 spin_unlock(&domain->lock);
603 }
604
605 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
606 }
607
608 /*
609 * This function uses heavy locking and may disable irqs for some time. But
610 * this is no issue because it is only called during resume.
611 */
612 void amd_iommu_flush_all_domains(void)
613 {
614 struct protection_domain *domain;
615 unsigned long flags;
616
617 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
618
619 list_for_each_entry(domain, &amd_iommu_pd_list, list) {
620 spin_lock(&domain->lock);
621 iommu_flush_tlb_pde(domain);
622 iommu_flush_complete(domain);
623 spin_unlock(&domain->lock);
624 }
625
626 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
627 }
628
629 static void flush_all_devices_for_iommu(struct amd_iommu *iommu)
630 {
631 int i;
632
633 for (i = 0; i <= amd_iommu_last_bdf; ++i) {
634 if (iommu != amd_iommu_rlookup_table[i])
635 continue;
636
637 iommu_queue_inv_dev_entry(iommu, i);
638 iommu_completion_wait(iommu);
639 }
640 }
641
642 static void flush_devices_by_domain(struct protection_domain *domain)
643 {
644 struct amd_iommu *iommu;
645 int i;
646
647 for (i = 0; i <= amd_iommu_last_bdf; ++i) {
648 if ((domain == NULL && amd_iommu_pd_table[i] == NULL) ||
649 (amd_iommu_pd_table[i] != domain))
650 continue;
651
652 iommu = amd_iommu_rlookup_table[i];
653 if (!iommu)
654 continue;
655
656 iommu_queue_inv_dev_entry(iommu, i);
657 iommu_completion_wait(iommu);
658 }
659 }
660
661 static void reset_iommu_command_buffer(struct amd_iommu *iommu)
662 {
663 pr_err("AMD-Vi: Resetting IOMMU command buffer\n");
664
665 if (iommu->reset_in_progress)
666 panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n");
667
668 iommu->reset_in_progress = true;
669
670 amd_iommu_reset_cmd_buffer(iommu);
671 flush_all_devices_for_iommu(iommu);
672 flush_all_domains_on_iommu(iommu);
673
674 iommu->reset_in_progress = false;
675 }
676
677 void amd_iommu_flush_all_devices(void)
678 {
679 flush_devices_by_domain(NULL);
680 }
681
682 /****************************************************************************
683 *
684 * The functions below are used the create the page table mappings for
685 * unity mapped regions.
686 *
687 ****************************************************************************/
688
689 /*
690 * This function is used to add another level to an IO page table. Adding
691 * another level increases the size of the address space by 9 bits to a size up
692 * to 64 bits.
693 */
694 static bool increase_address_space(struct protection_domain *domain,
695 gfp_t gfp)
696 {
697 u64 *pte;
698
699 if (domain->mode == PAGE_MODE_6_LEVEL)
700 /* address space already 64 bit large */
701 return false;
702
703 pte = (void *)get_zeroed_page(gfp);
704 if (!pte)
705 return false;
706
707 *pte = PM_LEVEL_PDE(domain->mode,
708 virt_to_phys(domain->pt_root));
709 domain->pt_root = pte;
710 domain->mode += 1;
711 domain->updated = true;
712
713 return true;
714 }
715
716 static u64 *alloc_pte(struct protection_domain *domain,
717 unsigned long address,
718 int end_lvl,
719 u64 **pte_page,
720 gfp_t gfp)
721 {
722 u64 *pte, *page;
723 int level;
724
725 while (address > PM_LEVEL_SIZE(domain->mode))
726 increase_address_space(domain, gfp);
727
728 level = domain->mode - 1;
729 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
730
731 while (level > end_lvl) {
732 if (!IOMMU_PTE_PRESENT(*pte)) {
733 page = (u64 *)get_zeroed_page(gfp);
734 if (!page)
735 return NULL;
736 *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
737 }
738
739 level -= 1;
740
741 pte = IOMMU_PTE_PAGE(*pte);
742
743 if (pte_page && level == end_lvl)
744 *pte_page = pte;
745
746 pte = &pte[PM_LEVEL_INDEX(level, address)];
747 }
748
749 return pte;
750 }
751
752 /*
753 * This function checks if there is a PTE for a given dma address. If
754 * there is one, it returns the pointer to it.
755 */
756 static u64 *fetch_pte(struct protection_domain *domain,
757 unsigned long address, int map_size)
758 {
759 int level;
760 u64 *pte;
761
762 level = domain->mode - 1;
763 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
764
765 while (level > map_size) {
766 if (!IOMMU_PTE_PRESENT(*pte))
767 return NULL;
768
769 level -= 1;
770
771 pte = IOMMU_PTE_PAGE(*pte);
772 pte = &pte[PM_LEVEL_INDEX(level, address)];
773
774 if ((PM_PTE_LEVEL(*pte) == 0) && level != map_size) {
775 pte = NULL;
776 break;
777 }
778 }
779
780 return pte;
781 }
782
783 /*
784 * Generic mapping functions. It maps a physical address into a DMA
785 * address space. It allocates the page table pages if necessary.
786 * In the future it can be extended to a generic mapping function
787 * supporting all features of AMD IOMMU page tables like level skipping
788 * and full 64 bit address spaces.
789 */
790 static int iommu_map_page(struct protection_domain *dom,
791 unsigned long bus_addr,
792 unsigned long phys_addr,
793 int prot,
794 int map_size)
795 {
796 u64 __pte, *pte;
797
798 bus_addr = PAGE_ALIGN(bus_addr);
799 phys_addr = PAGE_ALIGN(phys_addr);
800
801 BUG_ON(!PM_ALIGNED(map_size, bus_addr));
802 BUG_ON(!PM_ALIGNED(map_size, phys_addr));
803
804 if (!(prot & IOMMU_PROT_MASK))
805 return -EINVAL;
806
807 pte = alloc_pte(dom, bus_addr, map_size, NULL, GFP_KERNEL);
808
809 if (IOMMU_PTE_PRESENT(*pte))
810 return -EBUSY;
811
812 __pte = phys_addr | IOMMU_PTE_P;
813 if (prot & IOMMU_PROT_IR)
814 __pte |= IOMMU_PTE_IR;
815 if (prot & IOMMU_PROT_IW)
816 __pte |= IOMMU_PTE_IW;
817
818 *pte = __pte;
819
820 update_domain(dom);
821
822 return 0;
823 }
824
825 static void iommu_unmap_page(struct protection_domain *dom,
826 unsigned long bus_addr, int map_size)
827 {
828 u64 *pte = fetch_pte(dom, bus_addr, map_size);
829
830 if (pte)
831 *pte = 0;
832 }
833
834 /*
835 * This function checks if a specific unity mapping entry is needed for
836 * this specific IOMMU.
837 */
838 static int iommu_for_unity_map(struct amd_iommu *iommu,
839 struct unity_map_entry *entry)
840 {
841 u16 bdf, i;
842
843 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
844 bdf = amd_iommu_alias_table[i];
845 if (amd_iommu_rlookup_table[bdf] == iommu)
846 return 1;
847 }
848
849 return 0;
850 }
851
852 /*
853 * This function actually applies the mapping to the page table of the
854 * dma_ops domain.
855 */
856 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
857 struct unity_map_entry *e)
858 {
859 u64 addr;
860 int ret;
861
862 for (addr = e->address_start; addr < e->address_end;
863 addr += PAGE_SIZE) {
864 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
865 PM_MAP_4k);
866 if (ret)
867 return ret;
868 /*
869 * if unity mapping is in aperture range mark the page
870 * as allocated in the aperture
871 */
872 if (addr < dma_dom->aperture_size)
873 __set_bit(addr >> PAGE_SHIFT,
874 dma_dom->aperture[0]->bitmap);
875 }
876
877 return 0;
878 }
879
880 /*
881 * Init the unity mappings for a specific IOMMU in the system
882 *
883 * Basically iterates over all unity mapping entries and applies them to
884 * the default domain DMA of that IOMMU if necessary.
885 */
886 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
887 {
888 struct unity_map_entry *entry;
889 int ret;
890
891 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
892 if (!iommu_for_unity_map(iommu, entry))
893 continue;
894 ret = dma_ops_unity_map(iommu->default_dom, entry);
895 if (ret)
896 return ret;
897 }
898
899 return 0;
900 }
901
902 /*
903 * Inits the unity mappings required for a specific device
904 */
905 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
906 u16 devid)
907 {
908 struct unity_map_entry *e;
909 int ret;
910
911 list_for_each_entry(e, &amd_iommu_unity_map, list) {
912 if (!(devid >= e->devid_start && devid <= e->devid_end))
913 continue;
914 ret = dma_ops_unity_map(dma_dom, e);
915 if (ret)
916 return ret;
917 }
918
919 return 0;
920 }
921
922 /****************************************************************************
923 *
924 * The next functions belong to the address allocator for the dma_ops
925 * interface functions. They work like the allocators in the other IOMMU
926 * drivers. Its basically a bitmap which marks the allocated pages in
927 * the aperture. Maybe it could be enhanced in the future to a more
928 * efficient allocator.
929 *
930 ****************************************************************************/
931
932 /*
933 * The address allocator core functions.
934 *
935 * called with domain->lock held
936 */
937
938 /*
939 * Used to reserve address ranges in the aperture (e.g. for exclusion
940 * ranges.
941 */
942 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
943 unsigned long start_page,
944 unsigned int pages)
945 {
946 unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
947
948 if (start_page + pages > last_page)
949 pages = last_page - start_page;
950
951 for (i = start_page; i < start_page + pages; ++i) {
952 int index = i / APERTURE_RANGE_PAGES;
953 int page = i % APERTURE_RANGE_PAGES;
954 __set_bit(page, dom->aperture[index]->bitmap);
955 }
956 }
957
958 /*
959 * This function is used to add a new aperture range to an existing
960 * aperture in case of dma_ops domain allocation or address allocation
961 * failure.
962 */
963 static int alloc_new_range(struct dma_ops_domain *dma_dom,
964 bool populate, gfp_t gfp)
965 {
966 int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
967 struct amd_iommu *iommu;
968 int i;
969
970 #ifdef CONFIG_IOMMU_STRESS
971 populate = false;
972 #endif
973
974 if (index >= APERTURE_MAX_RANGES)
975 return -ENOMEM;
976
977 dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
978 if (!dma_dom->aperture[index])
979 return -ENOMEM;
980
981 dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
982 if (!dma_dom->aperture[index]->bitmap)
983 goto out_free;
984
985 dma_dom->aperture[index]->offset = dma_dom->aperture_size;
986
987 if (populate) {
988 unsigned long address = dma_dom->aperture_size;
989 int i, num_ptes = APERTURE_RANGE_PAGES / 512;
990 u64 *pte, *pte_page;
991
992 for (i = 0; i < num_ptes; ++i) {
993 pte = alloc_pte(&dma_dom->domain, address, PM_MAP_4k,
994 &pte_page, gfp);
995 if (!pte)
996 goto out_free;
997
998 dma_dom->aperture[index]->pte_pages[i] = pte_page;
999
1000 address += APERTURE_RANGE_SIZE / 64;
1001 }
1002 }
1003
1004 dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1005
1006 /* Intialize the exclusion range if necessary */
1007 for_each_iommu(iommu) {
1008 if (iommu->exclusion_start &&
1009 iommu->exclusion_start >= dma_dom->aperture[index]->offset
1010 && iommu->exclusion_start < dma_dom->aperture_size) {
1011 unsigned long startpage;
1012 int pages = iommu_num_pages(iommu->exclusion_start,
1013 iommu->exclusion_length,
1014 PAGE_SIZE);
1015 startpage = iommu->exclusion_start >> PAGE_SHIFT;
1016 dma_ops_reserve_addresses(dma_dom, startpage, pages);
1017 }
1018 }
1019
1020 /*
1021 * Check for areas already mapped as present in the new aperture
1022 * range and mark those pages as reserved in the allocator. Such
1023 * mappings may already exist as a result of requested unity
1024 * mappings for devices.
1025 */
1026 for (i = dma_dom->aperture[index]->offset;
1027 i < dma_dom->aperture_size;
1028 i += PAGE_SIZE) {
1029 u64 *pte = fetch_pte(&dma_dom->domain, i, PM_MAP_4k);
1030 if (!pte || !IOMMU_PTE_PRESENT(*pte))
1031 continue;
1032
1033 dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);
1034 }
1035
1036 update_domain(&dma_dom->domain);
1037
1038 return 0;
1039
1040 out_free:
1041 update_domain(&dma_dom->domain);
1042
1043 free_page((unsigned long)dma_dom->aperture[index]->bitmap);
1044
1045 kfree(dma_dom->aperture[index]);
1046 dma_dom->aperture[index] = NULL;
1047
1048 return -ENOMEM;
1049 }
1050
1051 static unsigned long dma_ops_area_alloc(struct device *dev,
1052 struct dma_ops_domain *dom,
1053 unsigned int pages,
1054 unsigned long align_mask,
1055 u64 dma_mask,
1056 unsigned long start)
1057 {
1058 unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1059 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1060 int i = start >> APERTURE_RANGE_SHIFT;
1061 unsigned long boundary_size;
1062 unsigned long address = -1;
1063 unsigned long limit;
1064
1065 next_bit >>= PAGE_SHIFT;
1066
1067 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1068 PAGE_SIZE) >> PAGE_SHIFT;
1069
1070 for (;i < max_index; ++i) {
1071 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1072
1073 if (dom->aperture[i]->offset >= dma_mask)
1074 break;
1075
1076 limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1077 dma_mask >> PAGE_SHIFT);
1078
1079 address = iommu_area_alloc(dom->aperture[i]->bitmap,
1080 limit, next_bit, pages, 0,
1081 boundary_size, align_mask);
1082 if (address != -1) {
1083 address = dom->aperture[i]->offset +
1084 (address << PAGE_SHIFT);
1085 dom->next_address = address + (pages << PAGE_SHIFT);
1086 break;
1087 }
1088
1089 next_bit = 0;
1090 }
1091
1092 return address;
1093 }
1094
1095 static unsigned long dma_ops_alloc_addresses(struct device *dev,
1096 struct dma_ops_domain *dom,
1097 unsigned int pages,
1098 unsigned long align_mask,
1099 u64 dma_mask)
1100 {
1101 unsigned long address;
1102
1103 #ifdef CONFIG_IOMMU_STRESS
1104 dom->next_address = 0;
1105 dom->need_flush = true;
1106 #endif
1107
1108 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1109 dma_mask, dom->next_address);
1110
1111 if (address == -1) {
1112 dom->next_address = 0;
1113 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1114 dma_mask, 0);
1115 dom->need_flush = true;
1116 }
1117
1118 if (unlikely(address == -1))
1119 address = DMA_ERROR_CODE;
1120
1121 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1122
1123 return address;
1124 }
1125
1126 /*
1127 * The address free function.
1128 *
1129 * called with domain->lock held
1130 */
1131 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1132 unsigned long address,
1133 unsigned int pages)
1134 {
1135 unsigned i = address >> APERTURE_RANGE_SHIFT;
1136 struct aperture_range *range = dom->aperture[i];
1137
1138 BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1139
1140 #ifdef CONFIG_IOMMU_STRESS
1141 if (i < 4)
1142 return;
1143 #endif
1144
1145 if (address >= dom->next_address)
1146 dom->need_flush = true;
1147
1148 address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1149
1150 iommu_area_free(range->bitmap, address, pages);
1151
1152 }
1153
1154 /****************************************************************************
1155 *
1156 * The next functions belong to the domain allocation. A domain is
1157 * allocated for every IOMMU as the default domain. If device isolation
1158 * is enabled, every device get its own domain. The most important thing
1159 * about domains is the page table mapping the DMA address space they
1160 * contain.
1161 *
1162 ****************************************************************************/
1163
1164 /*
1165 * This function adds a protection domain to the global protection domain list
1166 */
1167 static void add_domain_to_list(struct protection_domain *domain)
1168 {
1169 unsigned long flags;
1170
1171 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1172 list_add(&domain->list, &amd_iommu_pd_list);
1173 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1174 }
1175
1176 /*
1177 * This function removes a protection domain to the global
1178 * protection domain list
1179 */
1180 static void del_domain_from_list(struct protection_domain *domain)
1181 {
1182 unsigned long flags;
1183
1184 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1185 list_del(&domain->list);
1186 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1187 }
1188
1189 static u16 domain_id_alloc(void)
1190 {
1191 unsigned long flags;
1192 int id;
1193
1194 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1195 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1196 BUG_ON(id == 0);
1197 if (id > 0 && id < MAX_DOMAIN_ID)
1198 __set_bit(id, amd_iommu_pd_alloc_bitmap);
1199 else
1200 id = 0;
1201 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1202
1203 return id;
1204 }
1205
1206 static void domain_id_free(int id)
1207 {
1208 unsigned long flags;
1209
1210 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1211 if (id > 0 && id < MAX_DOMAIN_ID)
1212 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1213 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1214 }
1215
1216 static void free_pagetable(struct protection_domain *domain)
1217 {
1218 int i, j;
1219 u64 *p1, *p2, *p3;
1220
1221 p1 = domain->pt_root;
1222
1223 if (!p1)
1224 return;
1225
1226 for (i = 0; i < 512; ++i) {
1227 if (!IOMMU_PTE_PRESENT(p1[i]))
1228 continue;
1229
1230 p2 = IOMMU_PTE_PAGE(p1[i]);
1231 for (j = 0; j < 512; ++j) {
1232 if (!IOMMU_PTE_PRESENT(p2[j]))
1233 continue;
1234 p3 = IOMMU_PTE_PAGE(p2[j]);
1235 free_page((unsigned long)p3);
1236 }
1237
1238 free_page((unsigned long)p2);
1239 }
1240
1241 free_page((unsigned long)p1);
1242
1243 domain->pt_root = NULL;
1244 }
1245
1246 /*
1247 * Free a domain, only used if something went wrong in the
1248 * allocation path and we need to free an already allocated page table
1249 */
1250 static void dma_ops_domain_free(struct dma_ops_domain *dom)
1251 {
1252 int i;
1253
1254 if (!dom)
1255 return;
1256
1257 del_domain_from_list(&dom->domain);
1258
1259 free_pagetable(&dom->domain);
1260
1261 for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
1262 if (!dom->aperture[i])
1263 continue;
1264 free_page((unsigned long)dom->aperture[i]->bitmap);
1265 kfree(dom->aperture[i]);
1266 }
1267
1268 kfree(dom);
1269 }
1270
1271 /*
1272 * Allocates a new protection domain usable for the dma_ops functions.
1273 * It also intializes the page table and the address allocator data
1274 * structures required for the dma_ops interface
1275 */
1276 static struct dma_ops_domain *dma_ops_domain_alloc(void)
1277 {
1278 struct dma_ops_domain *dma_dom;
1279
1280 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
1281 if (!dma_dom)
1282 return NULL;
1283
1284 spin_lock_init(&dma_dom->domain.lock);
1285
1286 dma_dom->domain.id = domain_id_alloc();
1287 if (dma_dom->domain.id == 0)
1288 goto free_dma_dom;
1289 INIT_LIST_HEAD(&dma_dom->domain.dev_list);
1290 dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
1291 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1292 dma_dom->domain.flags = PD_DMA_OPS_MASK;
1293 dma_dom->domain.priv = dma_dom;
1294 if (!dma_dom->domain.pt_root)
1295 goto free_dma_dom;
1296
1297 dma_dom->need_flush = false;
1298 dma_dom->target_dev = 0xffff;
1299
1300 add_domain_to_list(&dma_dom->domain);
1301
1302 if (alloc_new_range(dma_dom, true, GFP_KERNEL))
1303 goto free_dma_dom;
1304
1305 /*
1306 * mark the first page as allocated so we never return 0 as
1307 * a valid dma-address. So we can use 0 as error value
1308 */
1309 dma_dom->aperture[0]->bitmap[0] = 1;
1310 dma_dom->next_address = 0;
1311
1312
1313 return dma_dom;
1314
1315 free_dma_dom:
1316 dma_ops_domain_free(dma_dom);
1317
1318 return NULL;
1319 }
1320
1321 /*
1322 * little helper function to check whether a given protection domain is a
1323 * dma_ops domain
1324 */
1325 static bool dma_ops_domain(struct protection_domain *domain)
1326 {
1327 return domain->flags & PD_DMA_OPS_MASK;
1328 }
1329
1330 static void set_dte_entry(u16 devid, struct protection_domain *domain)
1331 {
1332 u64 pte_root = virt_to_phys(domain->pt_root);
1333
1334 BUG_ON(amd_iommu_pd_table[devid] != NULL);
1335
1336 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
1337 << DEV_ENTRY_MODE_SHIFT;
1338 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1339
1340 amd_iommu_dev_table[devid].data[2] = domain->id;
1341 amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
1342 amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
1343
1344 amd_iommu_pd_table[devid] = domain;
1345
1346 }
1347
1348 static void clear_dte_entry(u16 devid)
1349 {
1350 struct protection_domain *domain = amd_iommu_pd_table[devid];
1351
1352 BUG_ON(domain == NULL);
1353
1354 /* remove domain from the lookup table */
1355 amd_iommu_pd_table[devid] = NULL;
1356
1357 /* remove entry from the device table seen by the hardware */
1358 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
1359 amd_iommu_dev_table[devid].data[1] = 0;
1360 amd_iommu_dev_table[devid].data[2] = 0;
1361
1362 amd_iommu_apply_erratum_63(devid);
1363 }
1364
1365 static void do_attach(struct device *dev, struct protection_domain *domain)
1366 {
1367 struct iommu_dev_data *dev_data;
1368 struct amd_iommu *iommu;
1369 u16 devid;
1370
1371 devid = get_device_id(dev);
1372 iommu = amd_iommu_rlookup_table[devid];
1373 dev_data = get_dev_data(dev);
1374
1375 /* Update data structures */
1376 dev_data->domain = domain;
1377 list_add(&dev_data->list, &domain->dev_list);
1378 set_dte_entry(devid, domain);
1379
1380 /* Do reference counting */
1381 domain->dev_iommu[iommu->index] += 1;
1382 domain->dev_cnt += 1;
1383
1384 /* Flush the DTE entry */
1385 iommu_queue_inv_dev_entry(iommu, devid);
1386 }
1387
1388 static void do_detach(struct device *dev)
1389 {
1390 struct iommu_dev_data *dev_data;
1391 struct amd_iommu *iommu;
1392 u16 devid;
1393
1394 devid = get_device_id(dev);
1395 iommu = amd_iommu_rlookup_table[devid];
1396 dev_data = get_dev_data(dev);
1397
1398 /* decrease reference counters */
1399 dev_data->domain->dev_iommu[iommu->index] -= 1;
1400 dev_data->domain->dev_cnt -= 1;
1401
1402 /* Update data structures */
1403 dev_data->domain = NULL;
1404 list_del(&dev_data->list);
1405 clear_dte_entry(devid);
1406
1407 /* Flush the DTE entry */
1408 iommu_queue_inv_dev_entry(iommu, devid);
1409 }
1410
1411 /*
1412 * If a device is not yet associated with a domain, this function does
1413 * assigns it visible for the hardware
1414 */
1415 static int __attach_device(struct device *dev,
1416 struct protection_domain *domain)
1417 {
1418 struct iommu_dev_data *dev_data, *alias_data;
1419
1420 dev_data = get_dev_data(dev);
1421 alias_data = get_dev_data(dev_data->alias);
1422
1423 if (!alias_data)
1424 return -EINVAL;
1425
1426 /* lock domain */
1427 spin_lock(&domain->lock);
1428
1429 /* Some sanity checks */
1430 if (alias_data->domain != NULL &&
1431 alias_data->domain != domain)
1432 return -EBUSY;
1433
1434 if (dev_data->domain != NULL &&
1435 dev_data->domain != domain)
1436 return -EBUSY;
1437
1438 /* Do real assignment */
1439 if (dev_data->alias != dev) {
1440 alias_data = get_dev_data(dev_data->alias);
1441 if (alias_data->domain == NULL)
1442 do_attach(dev_data->alias, domain);
1443
1444 atomic_inc(&alias_data->bind);
1445 }
1446
1447 if (dev_data->domain == NULL)
1448 do_attach(dev, domain);
1449
1450 atomic_inc(&dev_data->bind);
1451
1452 /* ready */
1453 spin_unlock(&domain->lock);
1454
1455 return 0;
1456 }
1457
1458 /*
1459 * If a device is not yet associated with a domain, this function does
1460 * assigns it visible for the hardware
1461 */
1462 static int attach_device(struct device *dev,
1463 struct protection_domain *domain)
1464 {
1465 unsigned long flags;
1466 int ret;
1467
1468 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1469 ret = __attach_device(dev, domain);
1470 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1471
1472 /*
1473 * We might boot into a crash-kernel here. The crashed kernel
1474 * left the caches in the IOMMU dirty. So we have to flush
1475 * here to evict all dirty stuff.
1476 */
1477 iommu_flush_tlb_pde(domain);
1478
1479 return ret;
1480 }
1481
1482 /*
1483 * Removes a device from a protection domain (unlocked)
1484 */
1485 static void __detach_device(struct device *dev)
1486 {
1487 struct iommu_dev_data *dev_data = get_dev_data(dev);
1488 struct iommu_dev_data *alias_data;
1489 unsigned long flags;
1490
1491 BUG_ON(!dev_data->domain);
1492
1493 spin_lock_irqsave(&dev_data->domain->lock, flags);
1494
1495 if (dev_data->alias != dev) {
1496 alias_data = get_dev_data(dev_data->alias);
1497 if (atomic_dec_and_test(&alias_data->bind))
1498 do_detach(dev_data->alias);
1499 }
1500
1501 if (atomic_dec_and_test(&dev_data->bind))
1502 do_detach(dev);
1503
1504 spin_unlock_irqrestore(&dev_data->domain->lock, flags);
1505
1506 /*
1507 * If we run in passthrough mode the device must be assigned to the
1508 * passthrough domain if it is detached from any other domain
1509 */
1510 if (iommu_pass_through && dev_data->domain == NULL)
1511 __attach_device(dev, pt_domain);
1512 }
1513
1514 /*
1515 * Removes a device from a protection domain (with devtable_lock held)
1516 */
1517 static void detach_device(struct device *dev)
1518 {
1519 unsigned long flags;
1520
1521 /* lock device table */
1522 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1523 __detach_device(dev);
1524 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1525 }
1526
1527 /*
1528 * Find out the protection domain structure for a given PCI device. This
1529 * will give us the pointer to the page table root for example.
1530 */
1531 static struct protection_domain *domain_for_device(struct device *dev)
1532 {
1533 struct protection_domain *dom;
1534 struct iommu_dev_data *dev_data, *alias_data;
1535 unsigned long flags;
1536 u16 devid, alias;
1537
1538 devid = get_device_id(dev);
1539 alias = amd_iommu_alias_table[devid];
1540 dev_data = get_dev_data(dev);
1541 alias_data = get_dev_data(dev_data->alias);
1542 if (!alias_data)
1543 return NULL;
1544
1545 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
1546 dom = dev_data->domain;
1547 if (dom == NULL &&
1548 alias_data->domain != NULL) {
1549 __attach_device(dev, alias_data->domain);
1550 dom = alias_data->domain;
1551 }
1552
1553 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1554
1555 return dom;
1556 }
1557
1558 static int device_change_notifier(struct notifier_block *nb,
1559 unsigned long action, void *data)
1560 {
1561 struct device *dev = data;
1562 u16 devid;
1563 struct protection_domain *domain;
1564 struct dma_ops_domain *dma_domain;
1565 struct amd_iommu *iommu;
1566 unsigned long flags;
1567
1568 if (!check_device(dev))
1569 return 0;
1570
1571 devid = get_device_id(dev);
1572 iommu = amd_iommu_rlookup_table[devid];
1573
1574 switch (action) {
1575 case BUS_NOTIFY_UNBOUND_DRIVER:
1576
1577 domain = domain_for_device(dev);
1578
1579 if (!domain)
1580 goto out;
1581 if (iommu_pass_through)
1582 break;
1583 detach_device(dev);
1584 break;
1585 case BUS_NOTIFY_ADD_DEVICE:
1586
1587 iommu_init_device(dev);
1588
1589 domain = domain_for_device(dev);
1590
1591 /* allocate a protection domain if a device is added */
1592 dma_domain = find_protection_domain(devid);
1593 if (dma_domain)
1594 goto out;
1595 dma_domain = dma_ops_domain_alloc();
1596 if (!dma_domain)
1597 goto out;
1598 dma_domain->target_dev = devid;
1599
1600 spin_lock_irqsave(&iommu_pd_list_lock, flags);
1601 list_add_tail(&dma_domain->list, &iommu_pd_list);
1602 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
1603
1604 break;
1605 case BUS_NOTIFY_DEL_DEVICE:
1606
1607 iommu_uninit_device(dev);
1608
1609 default:
1610 goto out;
1611 }
1612
1613 iommu_queue_inv_dev_entry(iommu, devid);
1614 iommu_completion_wait(iommu);
1615
1616 out:
1617 return 0;
1618 }
1619
1620 static struct notifier_block device_nb = {
1621 .notifier_call = device_change_notifier,
1622 };
1623
1624 /*****************************************************************************
1625 *
1626 * The next functions belong to the dma_ops mapping/unmapping code.
1627 *
1628 *****************************************************************************/
1629
1630 /*
1631 * In the dma_ops path we only have the struct device. This function
1632 * finds the corresponding IOMMU, the protection domain and the
1633 * requestor id for a given device.
1634 * If the device is not yet associated with a domain this is also done
1635 * in this function.
1636 */
1637 static struct protection_domain *get_domain(struct device *dev)
1638 {
1639 struct protection_domain *domain;
1640 struct dma_ops_domain *dma_dom;
1641 u16 devid = get_device_id(dev);
1642
1643 if (!check_device(dev))
1644 return ERR_PTR(-EINVAL);
1645
1646 domain = domain_for_device(dev);
1647 if (domain != NULL && !dma_ops_domain(domain))
1648 return ERR_PTR(-EBUSY);
1649
1650 if (domain != NULL)
1651 return domain;
1652
1653 /* Device not bount yet - bind it */
1654 dma_dom = find_protection_domain(devid);
1655 if (!dma_dom)
1656 dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
1657 attach_device(dev, &dma_dom->domain);
1658 DUMP_printk("Using protection domain %d for device %s\n",
1659 dma_dom->domain.id, dev_name(dev));
1660
1661 return &dma_dom->domain;
1662 }
1663
1664 static void update_device_table(struct protection_domain *domain)
1665 {
1666 unsigned long flags;
1667 int i;
1668
1669 for (i = 0; i <= amd_iommu_last_bdf; ++i) {
1670 if (amd_iommu_pd_table[i] != domain)
1671 continue;
1672 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1673 set_dte_entry(i, domain);
1674 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1675 }
1676 }
1677
1678 static void update_domain(struct protection_domain *domain)
1679 {
1680 if (!domain->updated)
1681 return;
1682
1683 update_device_table(domain);
1684 flush_devices_by_domain(domain);
1685 iommu_flush_tlb_pde(domain);
1686
1687 domain->updated = false;
1688 }
1689
1690 /*
1691 * This function fetches the PTE for a given address in the aperture
1692 */
1693 static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
1694 unsigned long address)
1695 {
1696 struct aperture_range *aperture;
1697 u64 *pte, *pte_page;
1698
1699 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1700 if (!aperture)
1701 return NULL;
1702
1703 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1704 if (!pte) {
1705 pte = alloc_pte(&dom->domain, address, PM_MAP_4k, &pte_page,
1706 GFP_ATOMIC);
1707 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
1708 } else
1709 pte += PM_LEVEL_INDEX(0, address);
1710
1711 update_domain(&dom->domain);
1712
1713 return pte;
1714 }
1715
1716 /*
1717 * This is the generic map function. It maps one 4kb page at paddr to
1718 * the given address in the DMA address space for the domain.
1719 */
1720 static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
1721 unsigned long address,
1722 phys_addr_t paddr,
1723 int direction)
1724 {
1725 u64 *pte, __pte;
1726
1727 WARN_ON(address > dom->aperture_size);
1728
1729 paddr &= PAGE_MASK;
1730
1731 pte = dma_ops_get_pte(dom, address);
1732 if (!pte)
1733 return DMA_ERROR_CODE;
1734
1735 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
1736
1737 if (direction == DMA_TO_DEVICE)
1738 __pte |= IOMMU_PTE_IR;
1739 else if (direction == DMA_FROM_DEVICE)
1740 __pte |= IOMMU_PTE_IW;
1741 else if (direction == DMA_BIDIRECTIONAL)
1742 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
1743
1744 WARN_ON(*pte);
1745
1746 *pte = __pte;
1747
1748 return (dma_addr_t)address;
1749 }
1750
1751 /*
1752 * The generic unmapping function for on page in the DMA address space.
1753 */
1754 static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
1755 unsigned long address)
1756 {
1757 struct aperture_range *aperture;
1758 u64 *pte;
1759
1760 if (address >= dom->aperture_size)
1761 return;
1762
1763 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1764 if (!aperture)
1765 return;
1766
1767 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1768 if (!pte)
1769 return;
1770
1771 pte += PM_LEVEL_INDEX(0, address);
1772
1773 WARN_ON(!*pte);
1774
1775 *pte = 0ULL;
1776 }
1777
1778 /*
1779 * This function contains common code for mapping of a physically
1780 * contiguous memory region into DMA address space. It is used by all
1781 * mapping functions provided with this IOMMU driver.
1782 * Must be called with the domain lock held.
1783 */
1784 static dma_addr_t __map_single(struct device *dev,
1785 struct dma_ops_domain *dma_dom,
1786 phys_addr_t paddr,
1787 size_t size,
1788 int dir,
1789 bool align,
1790 u64 dma_mask)
1791 {
1792 dma_addr_t offset = paddr & ~PAGE_MASK;
1793 dma_addr_t address, start, ret;
1794 unsigned int pages;
1795 unsigned long align_mask = 0;
1796 int i;
1797
1798 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
1799 paddr &= PAGE_MASK;
1800
1801 INC_STATS_COUNTER(total_map_requests);
1802
1803 if (pages > 1)
1804 INC_STATS_COUNTER(cross_page);
1805
1806 if (align)
1807 align_mask = (1UL << get_order(size)) - 1;
1808
1809 retry:
1810 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
1811 dma_mask);
1812 if (unlikely(address == DMA_ERROR_CODE)) {
1813 /*
1814 * setting next_address here will let the address
1815 * allocator only scan the new allocated range in the
1816 * first run. This is a small optimization.
1817 */
1818 dma_dom->next_address = dma_dom->aperture_size;
1819
1820 if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
1821 goto out;
1822
1823 /*
1824 * aperture was sucessfully enlarged by 128 MB, try
1825 * allocation again
1826 */
1827 goto retry;
1828 }
1829
1830 start = address;
1831 for (i = 0; i < pages; ++i) {
1832 ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
1833 if (ret == DMA_ERROR_CODE)
1834 goto out_unmap;
1835
1836 paddr += PAGE_SIZE;
1837 start += PAGE_SIZE;
1838 }
1839 address += offset;
1840
1841 ADD_STATS_COUNTER(alloced_io_mem, size);
1842
1843 if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
1844 iommu_flush_tlb(&dma_dom->domain);
1845 dma_dom->need_flush = false;
1846 } else if (unlikely(amd_iommu_np_cache))
1847 iommu_flush_pages(&dma_dom->domain, address, size);
1848
1849 out:
1850 return address;
1851
1852 out_unmap:
1853
1854 for (--i; i >= 0; --i) {
1855 start -= PAGE_SIZE;
1856 dma_ops_domain_unmap(dma_dom, start);
1857 }
1858
1859 dma_ops_free_addresses(dma_dom, address, pages);
1860
1861 return DMA_ERROR_CODE;
1862 }
1863
1864 /*
1865 * Does the reverse of the __map_single function. Must be called with
1866 * the domain lock held too
1867 */
1868 static void __unmap_single(struct dma_ops_domain *dma_dom,
1869 dma_addr_t dma_addr,
1870 size_t size,
1871 int dir)
1872 {
1873 dma_addr_t i, start;
1874 unsigned int pages;
1875
1876 if ((dma_addr == DMA_ERROR_CODE) ||
1877 (dma_addr + size > dma_dom->aperture_size))
1878 return;
1879
1880 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
1881 dma_addr &= PAGE_MASK;
1882 start = dma_addr;
1883
1884 for (i = 0; i < pages; ++i) {
1885 dma_ops_domain_unmap(dma_dom, start);
1886 start += PAGE_SIZE;
1887 }
1888
1889 SUB_STATS_COUNTER(alloced_io_mem, size);
1890
1891 dma_ops_free_addresses(dma_dom, dma_addr, pages);
1892
1893 if (amd_iommu_unmap_flush || dma_dom->need_flush) {
1894 iommu_flush_pages(&dma_dom->domain, dma_addr, size);
1895 dma_dom->need_flush = false;
1896 }
1897 }
1898
1899 /*
1900 * The exported map_single function for dma_ops.
1901 */
1902 static dma_addr_t map_page(struct device *dev, struct page *page,
1903 unsigned long offset, size_t size,
1904 enum dma_data_direction dir,
1905 struct dma_attrs *attrs)
1906 {
1907 unsigned long flags;
1908 struct protection_domain *domain;
1909 dma_addr_t addr;
1910 u64 dma_mask;
1911 phys_addr_t paddr = page_to_phys(page) + offset;
1912
1913 INC_STATS_COUNTER(cnt_map_single);
1914
1915 domain = get_domain(dev);
1916 if (PTR_ERR(domain) == -EINVAL)
1917 return (dma_addr_t)paddr;
1918 else if (IS_ERR(domain))
1919 return DMA_ERROR_CODE;
1920
1921 dma_mask = *dev->dma_mask;
1922
1923 spin_lock_irqsave(&domain->lock, flags);
1924
1925 addr = __map_single(dev, domain->priv, paddr, size, dir, false,
1926 dma_mask);
1927 if (addr == DMA_ERROR_CODE)
1928 goto out;
1929
1930 iommu_flush_complete(domain);
1931
1932 out:
1933 spin_unlock_irqrestore(&domain->lock, flags);
1934
1935 return addr;
1936 }
1937
1938 /*
1939 * The exported unmap_single function for dma_ops.
1940 */
1941 static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
1942 enum dma_data_direction dir, struct dma_attrs *attrs)
1943 {
1944 unsigned long flags;
1945 struct protection_domain *domain;
1946
1947 INC_STATS_COUNTER(cnt_unmap_single);
1948
1949 domain = get_domain(dev);
1950 if (IS_ERR(domain))
1951 return;
1952
1953 spin_lock_irqsave(&domain->lock, flags);
1954
1955 __unmap_single(domain->priv, dma_addr, size, dir);
1956
1957 iommu_flush_complete(domain);
1958
1959 spin_unlock_irqrestore(&domain->lock, flags);
1960 }
1961
1962 /*
1963 * This is a special map_sg function which is used if we should map a
1964 * device which is not handled by an AMD IOMMU in the system.
1965 */
1966 static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
1967 int nelems, int dir)
1968 {
1969 struct scatterlist *s;
1970 int i;
1971
1972 for_each_sg(sglist, s, nelems, i) {
1973 s->dma_address = (dma_addr_t)sg_phys(s);
1974 s->dma_length = s->length;
1975 }
1976
1977 return nelems;
1978 }
1979
1980 /*
1981 * The exported map_sg function for dma_ops (handles scatter-gather
1982 * lists).
1983 */
1984 static int map_sg(struct device *dev, struct scatterlist *sglist,
1985 int nelems, enum dma_data_direction dir,
1986 struct dma_attrs *attrs)
1987 {
1988 unsigned long flags;
1989 struct protection_domain *domain;
1990 int i;
1991 struct scatterlist *s;
1992 phys_addr_t paddr;
1993 int mapped_elems = 0;
1994 u64 dma_mask;
1995
1996 INC_STATS_COUNTER(cnt_map_sg);
1997
1998 domain = get_domain(dev);
1999 if (PTR_ERR(domain) == -EINVAL)
2000 return map_sg_no_iommu(dev, sglist, nelems, dir);
2001 else if (IS_ERR(domain))
2002 return 0;
2003
2004 dma_mask = *dev->dma_mask;
2005
2006 spin_lock_irqsave(&domain->lock, flags);
2007
2008 for_each_sg(sglist, s, nelems, i) {
2009 paddr = sg_phys(s);
2010
2011 s->dma_address = __map_single(dev, domain->priv,
2012 paddr, s->length, dir, false,
2013 dma_mask);
2014
2015 if (s->dma_address) {
2016 s->dma_length = s->length;
2017 mapped_elems++;
2018 } else
2019 goto unmap;
2020 }
2021
2022 iommu_flush_complete(domain);
2023
2024 out:
2025 spin_unlock_irqrestore(&domain->lock, flags);
2026
2027 return mapped_elems;
2028 unmap:
2029 for_each_sg(sglist, s, mapped_elems, i) {
2030 if (s->dma_address)
2031 __unmap_single(domain->priv, s->dma_address,
2032 s->dma_length, dir);
2033 s->dma_address = s->dma_length = 0;
2034 }
2035
2036 mapped_elems = 0;
2037
2038 goto out;
2039 }
2040
2041 /*
2042 * The exported map_sg function for dma_ops (handles scatter-gather
2043 * lists).
2044 */
2045 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2046 int nelems, enum dma_data_direction dir,
2047 struct dma_attrs *attrs)
2048 {
2049 unsigned long flags;
2050 struct protection_domain *domain;
2051 struct scatterlist *s;
2052 int i;
2053
2054 INC_STATS_COUNTER(cnt_unmap_sg);
2055
2056 domain = get_domain(dev);
2057 if (IS_ERR(domain))
2058 return;
2059
2060 spin_lock_irqsave(&domain->lock, flags);
2061
2062 for_each_sg(sglist, s, nelems, i) {
2063 __unmap_single(domain->priv, s->dma_address,
2064 s->dma_length, dir);
2065 s->dma_address = s->dma_length = 0;
2066 }
2067
2068 iommu_flush_complete(domain);
2069
2070 spin_unlock_irqrestore(&domain->lock, flags);
2071 }
2072
2073 /*
2074 * The exported alloc_coherent function for dma_ops.
2075 */
2076 static void *alloc_coherent(struct device *dev, size_t size,
2077 dma_addr_t *dma_addr, gfp_t flag)
2078 {
2079 unsigned long flags;
2080 void *virt_addr;
2081 struct protection_domain *domain;
2082 phys_addr_t paddr;
2083 u64 dma_mask = dev->coherent_dma_mask;
2084
2085 INC_STATS_COUNTER(cnt_alloc_coherent);
2086
2087 domain = get_domain(dev);
2088 if (PTR_ERR(domain) == -EINVAL) {
2089 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2090 *dma_addr = __pa(virt_addr);
2091 return virt_addr;
2092 } else if (IS_ERR(domain))
2093 return NULL;
2094
2095 dma_mask = dev->coherent_dma_mask;
2096 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2097 flag |= __GFP_ZERO;
2098
2099 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2100 if (!virt_addr)
2101 return NULL;
2102
2103 paddr = virt_to_phys(virt_addr);
2104
2105 if (!dma_mask)
2106 dma_mask = *dev->dma_mask;
2107
2108 spin_lock_irqsave(&domain->lock, flags);
2109
2110 *dma_addr = __map_single(dev, domain->priv, paddr,
2111 size, DMA_BIDIRECTIONAL, true, dma_mask);
2112
2113 if (*dma_addr == DMA_ERROR_CODE) {
2114 spin_unlock_irqrestore(&domain->lock, flags);
2115 goto out_free;
2116 }
2117
2118 iommu_flush_complete(domain);
2119
2120 spin_unlock_irqrestore(&domain->lock, flags);
2121
2122 return virt_addr;
2123
2124 out_free:
2125
2126 free_pages((unsigned long)virt_addr, get_order(size));
2127
2128 return NULL;
2129 }
2130
2131 /*
2132 * The exported free_coherent function for dma_ops.
2133 */
2134 static void free_coherent(struct device *dev, size_t size,
2135 void *virt_addr, dma_addr_t dma_addr)
2136 {
2137 unsigned long flags;
2138 struct protection_domain *domain;
2139
2140 INC_STATS_COUNTER(cnt_free_coherent);
2141
2142 domain = get_domain(dev);
2143 if (IS_ERR(domain))
2144 goto free_mem;
2145
2146 spin_lock_irqsave(&domain->lock, flags);
2147
2148 __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2149
2150 iommu_flush_complete(domain);
2151
2152 spin_unlock_irqrestore(&domain->lock, flags);
2153
2154 free_mem:
2155 free_pages((unsigned long)virt_addr, get_order(size));
2156 }
2157
2158 /*
2159 * This function is called by the DMA layer to find out if we can handle a
2160 * particular device. It is part of the dma_ops.
2161 */
2162 static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2163 {
2164 return check_device(dev);
2165 }
2166
2167 /*
2168 * The function for pre-allocating protection domains.
2169 *
2170 * If the driver core informs the DMA layer if a driver grabs a device
2171 * we don't need to preallocate the protection domains anymore.
2172 * For now we have to.
2173 */
2174 static void prealloc_protection_domains(void)
2175 {
2176 struct pci_dev *dev = NULL;
2177 struct dma_ops_domain *dma_dom;
2178 u16 devid;
2179
2180 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
2181
2182 /* Do we handle this device? */
2183 if (!check_device(&dev->dev))
2184 continue;
2185
2186 iommu_init_device(&dev->dev);
2187
2188 /* Is there already any domain for it? */
2189 if (domain_for_device(&dev->dev))
2190 continue;
2191
2192 devid = get_device_id(&dev->dev);
2193
2194 dma_dom = dma_ops_domain_alloc();
2195 if (!dma_dom)
2196 continue;
2197 init_unity_mappings_for_device(dma_dom, devid);
2198 dma_dom->target_dev = devid;
2199
2200 attach_device(&dev->dev, &dma_dom->domain);
2201
2202 list_add_tail(&dma_dom->list, &iommu_pd_list);
2203 }
2204 }
2205
2206 static struct dma_map_ops amd_iommu_dma_ops = {
2207 .alloc_coherent = alloc_coherent,
2208 .free_coherent = free_coherent,
2209 .map_page = map_page,
2210 .unmap_page = unmap_page,
2211 .map_sg = map_sg,
2212 .unmap_sg = unmap_sg,
2213 .dma_supported = amd_iommu_dma_supported,
2214 };
2215
2216 /*
2217 * The function which clues the AMD IOMMU driver into dma_ops.
2218 */
2219 int __init amd_iommu_init_dma_ops(void)
2220 {
2221 struct amd_iommu *iommu;
2222 int ret;
2223
2224 /*
2225 * first allocate a default protection domain for every IOMMU we
2226 * found in the system. Devices not assigned to any other
2227 * protection domain will be assigned to the default one.
2228 */
2229 for_each_iommu(iommu) {
2230 iommu->default_dom = dma_ops_domain_alloc();
2231 if (iommu->default_dom == NULL)
2232 return -ENOMEM;
2233 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
2234 ret = iommu_init_unity_mappings(iommu);
2235 if (ret)
2236 goto free_domains;
2237 }
2238
2239 /*
2240 * Pre-allocate the protection domains for each device.
2241 */
2242 prealloc_protection_domains();
2243
2244 iommu_detected = 1;
2245 swiotlb = 0;
2246 #ifdef CONFIG_GART_IOMMU
2247 gart_iommu_aperture_disabled = 1;
2248 gart_iommu_aperture = 0;
2249 #endif
2250
2251 /* Make the driver finally visible to the drivers */
2252 dma_ops = &amd_iommu_dma_ops;
2253
2254 register_iommu(&amd_iommu_ops);
2255
2256 bus_register_notifier(&pci_bus_type, &device_nb);
2257
2258 amd_iommu_stats_init();
2259
2260 return 0;
2261
2262 free_domains:
2263
2264 for_each_iommu(iommu) {
2265 if (iommu->default_dom)
2266 dma_ops_domain_free(iommu->default_dom);
2267 }
2268
2269 return ret;
2270 }
2271
2272 /*****************************************************************************
2273 *
2274 * The following functions belong to the exported interface of AMD IOMMU
2275 *
2276 * This interface allows access to lower level functions of the IOMMU
2277 * like protection domain handling and assignement of devices to domains
2278 * which is not possible with the dma_ops interface.
2279 *
2280 *****************************************************************************/
2281
2282 static void cleanup_domain(struct protection_domain *domain)
2283 {
2284 unsigned long flags;
2285 u16 devid;
2286
2287 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2288
2289 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
2290 if (amd_iommu_pd_table[devid] == domain)
2291 clear_dte_entry(devid);
2292
2293 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2294 }
2295
2296 static void protection_domain_free(struct protection_domain *domain)
2297 {
2298 if (!domain)
2299 return;
2300
2301 del_domain_from_list(domain);
2302
2303 if (domain->id)
2304 domain_id_free(domain->id);
2305
2306 kfree(domain);
2307 }
2308
2309 static struct protection_domain *protection_domain_alloc(void)
2310 {
2311 struct protection_domain *domain;
2312
2313 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2314 if (!domain)
2315 return NULL;
2316
2317 spin_lock_init(&domain->lock);
2318 domain->id = domain_id_alloc();
2319 if (!domain->id)
2320 goto out_err;
2321 INIT_LIST_HEAD(&domain->dev_list);
2322
2323 add_domain_to_list(domain);
2324
2325 return domain;
2326
2327 out_err:
2328 kfree(domain);
2329
2330 return NULL;
2331 }
2332
2333 static int amd_iommu_domain_init(struct iommu_domain *dom)
2334 {
2335 struct protection_domain *domain;
2336
2337 domain = protection_domain_alloc();
2338 if (!domain)
2339 goto out_free;
2340
2341 domain->mode = PAGE_MODE_3_LEVEL;
2342 domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2343 if (!domain->pt_root)
2344 goto out_free;
2345
2346 dom->priv = domain;
2347
2348 return 0;
2349
2350 out_free:
2351 protection_domain_free(domain);
2352
2353 return -ENOMEM;
2354 }
2355
2356 static void amd_iommu_domain_destroy(struct iommu_domain *dom)
2357 {
2358 struct protection_domain *domain = dom->priv;
2359
2360 if (!domain)
2361 return;
2362
2363 if (domain->dev_cnt > 0)
2364 cleanup_domain(domain);
2365
2366 BUG_ON(domain->dev_cnt != 0);
2367
2368 free_pagetable(domain);
2369
2370 domain_id_free(domain->id);
2371
2372 kfree(domain);
2373
2374 dom->priv = NULL;
2375 }
2376
2377 static void amd_iommu_detach_device(struct iommu_domain *dom,
2378 struct device *dev)
2379 {
2380 struct iommu_dev_data *dev_data = dev->archdata.iommu;
2381 struct amd_iommu *iommu;
2382 u16 devid;
2383
2384 if (!check_device(dev))
2385 return;
2386
2387 devid = get_device_id(dev);
2388
2389 if (dev_data->domain != NULL)
2390 detach_device(dev);
2391
2392 iommu = amd_iommu_rlookup_table[devid];
2393 if (!iommu)
2394 return;
2395
2396 iommu_queue_inv_dev_entry(iommu, devid);
2397 iommu_completion_wait(iommu);
2398 }
2399
2400 static int amd_iommu_attach_device(struct iommu_domain *dom,
2401 struct device *dev)
2402 {
2403 struct protection_domain *domain = dom->priv;
2404 struct iommu_dev_data *dev_data;
2405 struct amd_iommu *iommu;
2406 int ret;
2407 u16 devid;
2408
2409 if (!check_device(dev))
2410 return -EINVAL;
2411
2412 dev_data = dev->archdata.iommu;
2413
2414 devid = get_device_id(dev);
2415
2416 iommu = amd_iommu_rlookup_table[devid];
2417 if (!iommu)
2418 return -EINVAL;
2419
2420 if (dev_data->domain)
2421 detach_device(dev);
2422
2423 ret = attach_device(dev, domain);
2424
2425 iommu_completion_wait(iommu);
2426
2427 return ret;
2428 }
2429
2430 static int amd_iommu_map_range(struct iommu_domain *dom,
2431 unsigned long iova, phys_addr_t paddr,
2432 size_t size, int iommu_prot)
2433 {
2434 struct protection_domain *domain = dom->priv;
2435 unsigned long i, npages = iommu_num_pages(paddr, size, PAGE_SIZE);
2436 int prot = 0;
2437 int ret;
2438
2439 if (iommu_prot & IOMMU_READ)
2440 prot |= IOMMU_PROT_IR;
2441 if (iommu_prot & IOMMU_WRITE)
2442 prot |= IOMMU_PROT_IW;
2443
2444 iova &= PAGE_MASK;
2445 paddr &= PAGE_MASK;
2446
2447 for (i = 0; i < npages; ++i) {
2448 ret = iommu_map_page(domain, iova, paddr, prot, PM_MAP_4k);
2449 if (ret)
2450 return ret;
2451
2452 iova += PAGE_SIZE;
2453 paddr += PAGE_SIZE;
2454 }
2455
2456 return 0;
2457 }
2458
2459 static void amd_iommu_unmap_range(struct iommu_domain *dom,
2460 unsigned long iova, size_t size)
2461 {
2462
2463 struct protection_domain *domain = dom->priv;
2464 unsigned long i, npages = iommu_num_pages(iova, size, PAGE_SIZE);
2465
2466 iova &= PAGE_MASK;
2467
2468 for (i = 0; i < npages; ++i) {
2469 iommu_unmap_page(domain, iova, PM_MAP_4k);
2470 iova += PAGE_SIZE;
2471 }
2472
2473 iommu_flush_tlb_pde(domain);
2474 }
2475
2476 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2477 unsigned long iova)
2478 {
2479 struct protection_domain *domain = dom->priv;
2480 unsigned long offset = iova & ~PAGE_MASK;
2481 phys_addr_t paddr;
2482 u64 *pte;
2483
2484 pte = fetch_pte(domain, iova, PM_MAP_4k);
2485
2486 if (!pte || !IOMMU_PTE_PRESENT(*pte))
2487 return 0;
2488
2489 paddr = *pte & IOMMU_PAGE_MASK;
2490 paddr |= offset;
2491
2492 return paddr;
2493 }
2494
2495 static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
2496 unsigned long cap)
2497 {
2498 return 0;
2499 }
2500
2501 static struct iommu_ops amd_iommu_ops = {
2502 .domain_init = amd_iommu_domain_init,
2503 .domain_destroy = amd_iommu_domain_destroy,
2504 .attach_dev = amd_iommu_attach_device,
2505 .detach_dev = amd_iommu_detach_device,
2506 .map = amd_iommu_map_range,
2507 .unmap = amd_iommu_unmap_range,
2508 .iova_to_phys = amd_iommu_iova_to_phys,
2509 .domain_has_cap = amd_iommu_domain_has_cap,
2510 };
2511
2512 /*****************************************************************************
2513 *
2514 * The next functions do a basic initialization of IOMMU for pass through
2515 * mode
2516 *
2517 * In passthrough mode the IOMMU is initialized and enabled but not used for
2518 * DMA-API translation.
2519 *
2520 *****************************************************************************/
2521
2522 int __init amd_iommu_init_passthrough(void)
2523 {
2524 struct amd_iommu *iommu;
2525 struct pci_dev *dev = NULL;
2526 u16 devid;
2527
2528 /* allocate passthroug domain */
2529 pt_domain = protection_domain_alloc();
2530 if (!pt_domain)
2531 return -ENOMEM;
2532
2533 pt_domain->mode |= PAGE_MODE_NONE;
2534
2535 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
2536
2537 if (!check_device(&dev->dev))
2538 continue;
2539
2540 devid = get_device_id(&dev->dev);
2541
2542 iommu = amd_iommu_rlookup_table[devid];
2543 if (!iommu)
2544 continue;
2545
2546 attach_device(&dev->dev, pt_domain);
2547 }
2548
2549 pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
2550
2551 return 0;
2552 }
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree