?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/radeon: add runtime PM support (v2)
[linux-2.6.git] / drivers / iommu / fsl_pamu_domain.c
blobc857c30da9791e9a01f9630f293e5a0d1315dc47
1 /*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 *
15 * Copyright (C) 2013 Freescale Semiconductor, Inc.
16 * Author: Varun Sethi <varun.sethi@freescale.com>
17 *
18 */
19
20 #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__
21
22 #include <linux/init.h>
23 #include <linux/iommu.h>
24 #include <linux/notifier.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/types.h>
28 #include <linux/mm.h>
29 #include <linux/interrupt.h>
30 #include <linux/device.h>
31 #include <linux/of_platform.h>
32 #include <linux/bootmem.h>
33 #include <linux/err.h>
34 #include <asm/io.h>
35 #include <asm/bitops.h>
36
37 #include <asm/pci-bridge.h>
38 #include <sysdev/fsl_pci.h>
39
40 #include "fsl_pamu_domain.h"
41 #include "pci.h"
42
43 /*
44 * Global spinlock that needs to be held while
45 * configuring PAMU.
46 */
47 static DEFINE_SPINLOCK(iommu_lock);
48
49 static struct kmem_cache *fsl_pamu_domain_cache;
50 static struct kmem_cache *iommu_devinfo_cache;
51 static DEFINE_SPINLOCK(device_domain_lock);
52
53 static int __init iommu_init_mempool(void)
54 {
55
56 fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
57 sizeof(struct fsl_dma_domain),
58 0,
59 SLAB_HWCACHE_ALIGN,
60
61 NULL);
62 if (!fsl_pamu_domain_cache) {
63 pr_debug("Couldn't create fsl iommu_domain cache\n");
64 return -ENOMEM;
65 }
66
67 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
68 sizeof(struct device_domain_info),
69 0,
70 SLAB_HWCACHE_ALIGN,
71 NULL);
72 if (!iommu_devinfo_cache) {
73 pr_debug("Couldn't create devinfo cache\n");
74 kmem_cache_destroy(fsl_pamu_domain_cache);
75 return -ENOMEM;
76 }
77
78 return 0;
79 }
80
81 static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova)
82 {
83 u32 win_cnt = dma_domain->win_cnt;
84 struct dma_window *win_ptr =
85 &dma_domain->win_arr[0];
86 struct iommu_domain_geometry *geom;
87
88 geom = &dma_domain->iommu_domain->geometry;
89
90 if (!win_cnt || !dma_domain->geom_size) {
91 pr_debug("Number of windows/geometry not configured for the domain\n");
92 return 0;
93 }
94
95 if (win_cnt > 1) {
96 u64 subwin_size;
97 dma_addr_t subwin_iova;
98 u32 wnd;
99
100 subwin_size = dma_domain->geom_size >> ilog2(win_cnt);
101 subwin_iova = iova & ~(subwin_size - 1);
102 wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size);
103 win_ptr = &dma_domain->win_arr[wnd];
104 }
105
106 if (win_ptr->valid)
107 return (win_ptr->paddr + (iova & (win_ptr->size - 1)));
108
109 return 0;
110 }
111
112 static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain)
113 {
114 struct dma_window *sub_win_ptr =
115 &dma_domain->win_arr[0];
116 int i, ret;
117 unsigned long rpn, flags;
118
119 for (i = 0; i < dma_domain->win_cnt; i++) {
120 if (sub_win_ptr[i].valid) {
121 rpn = sub_win_ptr[i].paddr >>
122 PAMU_PAGE_SHIFT;
123 spin_lock_irqsave(&iommu_lock, flags);
124 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i,
125 sub_win_ptr[i].size,
126 ~(u32)0,
127 rpn,
128 dma_domain->snoop_id,
129 dma_domain->stash_id,
130 (i > 0) ? 1 : 0,
131 sub_win_ptr[i].prot);
132 spin_unlock_irqrestore(&iommu_lock, flags);
133 if (ret) {
134 pr_debug("PAMU SPAACE configuration failed for liodn %d\n",
135 liodn);
136 return ret;
137 }
138 }
139 }
140
141 return ret;
142 }
143
144 static int map_win(int liodn, struct fsl_dma_domain *dma_domain)
145 {
146 int ret;
147 struct dma_window *wnd = &dma_domain->win_arr[0];
148 phys_addr_t wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
149 unsigned long flags;
150
151 spin_lock_irqsave(&iommu_lock, flags);
152 ret = pamu_config_ppaace(liodn, wnd_addr,
153 wnd->size,
154 ~(u32)0,
155 wnd->paddr >> PAMU_PAGE_SHIFT,
156 dma_domain->snoop_id, dma_domain->stash_id,
157 0, wnd->prot);
158 spin_unlock_irqrestore(&iommu_lock, flags);
159 if (ret)
160 pr_debug("PAMU PAACE configuration failed for liodn %d\n",
161 liodn);
162
163 return ret;
164 }
165
166 /* Map the DMA window corresponding to the LIODN */
167 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)
168 {
169 if (dma_domain->win_cnt > 1)
170 return map_subwins(liodn, dma_domain);
171 else
172 return map_win(liodn, dma_domain);
173
174 }
175
176 /* Update window/subwindow mapping for the LIODN */
177 static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr)
178 {
179 int ret;
180 struct dma_window *wnd = &dma_domain->win_arr[wnd_nr];
181 unsigned long flags;
182
183 spin_lock_irqsave(&iommu_lock, flags);
184 if (dma_domain->win_cnt > 1) {
185 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr,
186 wnd->size,
187 ~(u32)0,
188 wnd->paddr >> PAMU_PAGE_SHIFT,
189 dma_domain->snoop_id,
190 dma_domain->stash_id,
191 (wnd_nr > 0) ? 1 : 0,
192 wnd->prot);
193 if (ret)
194 pr_debug("Subwindow reconfiguration failed for liodn %d\n", liodn);
195 } else {
196 phys_addr_t wnd_addr;
197
198 wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
199
200 ret = pamu_config_ppaace(liodn, wnd_addr,
201 wnd->size,
202 ~(u32)0,
203 wnd->paddr >> PAMU_PAGE_SHIFT,
204 dma_domain->snoop_id, dma_domain->stash_id,
205 0, wnd->prot);
206 if (ret)
207 pr_debug("Window reconfiguration failed for liodn %d\n", liodn);
208 }
209
210 spin_unlock_irqrestore(&iommu_lock, flags);
211
212 return ret;
213 }
214
215 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
216 u32 val)
217 {
218 int ret = 0, i;
219 unsigned long flags;
220
221 spin_lock_irqsave(&iommu_lock, flags);
222 if (!dma_domain->win_arr) {
223 pr_debug("Windows not configured, stash destination update failed for liodn %d\n", liodn);
224 spin_unlock_irqrestore(&iommu_lock, flags);
225 return -EINVAL;
226 }
227
228 for (i = 0; i < dma_domain->win_cnt; i++) {
229 ret = pamu_update_paace_stash(liodn, i, val);
230 if (ret) {
231 pr_debug("Failed to update SPAACE %d field for liodn %d\n ", i, liodn);
232 spin_unlock_irqrestore(&iommu_lock, flags);
233 return ret;
234 }
235 }
236
237 spin_unlock_irqrestore(&iommu_lock, flags);
238
239 return ret;
240 }
241
242 /* Set the geometry parameters for a LIODN */
243 static int pamu_set_liodn(int liodn, struct device *dev,
244 struct fsl_dma_domain *dma_domain,
245 struct iommu_domain_geometry *geom_attr,
246 u32 win_cnt)
247 {
248 phys_addr_t window_addr, window_size;
249 phys_addr_t subwin_size;
250 int ret = 0, i;
251 u32 omi_index = ~(u32)0;
252 unsigned long flags;
253
254 /*
255 * Configure the omi_index at the geometry setup time.
256 * This is a static value which depends on the type of
257 * device and would not change thereafter.
258 */
259 get_ome_index(&omi_index, dev);
260
261 window_addr = geom_attr->aperture_start;
262 window_size = dma_domain->geom_size;
263
264 spin_lock_irqsave(&iommu_lock, flags);
265 ret = pamu_disable_liodn(liodn);
266 if (!ret)
267 ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index,
268 0, dma_domain->snoop_id,
269 dma_domain->stash_id, win_cnt, 0);
270 spin_unlock_irqrestore(&iommu_lock, flags);
271 if (ret) {
272 pr_debug("PAMU PAACE configuration failed for liodn %d, win_cnt =%d\n", liodn, win_cnt);
273 return ret;
274 }
275
276 if (win_cnt > 1) {
277 subwin_size = window_size >> ilog2(win_cnt);
278 for (i = 0; i < win_cnt; i++) {
279 spin_lock_irqsave(&iommu_lock, flags);
280 ret = pamu_disable_spaace(liodn, i);
281 if (!ret)
282 ret = pamu_config_spaace(liodn, win_cnt, i,
283 subwin_size, omi_index,
284 0, dma_domain->snoop_id,
285 dma_domain->stash_id,
286 0, 0);
287 spin_unlock_irqrestore(&iommu_lock, flags);
288 if (ret) {
289 pr_debug("PAMU SPAACE configuration failed for liodn %d\n", liodn);
290 return ret;
291 }
292 }
293 }
294
295 return ret;
296 }
297
298 static int check_size(u64 size, dma_addr_t iova)
299 {
300 /*
301 * Size must be a power of two and at least be equal
302 * to PAMU page size.
303 */
304 if (!is_power_of_2(size) || size < PAMU_PAGE_SIZE) {
305 pr_debug("%s: size too small or not a power of two\n", __func__);
306 return -EINVAL;
307 }
308
309 /* iova must be page size aligned*/
310 if (iova & (size - 1)) {
311 pr_debug("%s: address is not aligned with window size\n", __func__);
312 return -EINVAL;
313 }
314
315 return 0;
316 }
317
318 static struct fsl_dma_domain *iommu_alloc_dma_domain(void)
319 {
320 struct fsl_dma_domain *domain;
321
322 domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
323 if (!domain)
324 return NULL;
325
326 domain->stash_id = ~(u32)0;
327 domain->snoop_id = ~(u32)0;
328 domain->win_cnt = pamu_get_max_subwin_cnt();
329 domain->geom_size = 0;
330
331 INIT_LIST_HEAD(&domain->devices);
332
333 spin_lock_init(&domain->domain_lock);
334
335 return domain;
336 }
337
338 static inline struct device_domain_info *find_domain(struct device *dev)
339 {
340 return dev->archdata.iommu_domain;
341 }
342
343 static void remove_device_ref(struct device_domain_info *info, u32 win_cnt)
344 {
345 unsigned long flags;
346
347 list_del(&info->link);
348 spin_lock_irqsave(&iommu_lock, flags);
349 if (win_cnt > 1)
350 pamu_free_subwins(info->liodn);
351 pamu_disable_liodn(info->liodn);
352 spin_unlock_irqrestore(&iommu_lock, flags);
353 spin_lock_irqsave(&device_domain_lock, flags);
354 info->dev->archdata.iommu_domain = NULL;
355 kmem_cache_free(iommu_devinfo_cache, info);
356 spin_unlock_irqrestore(&device_domain_lock, flags);
357 }
358
359 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
360 {
361 struct device_domain_info *info, *tmp;
362 unsigned long flags;
363
364 spin_lock_irqsave(&dma_domain->domain_lock, flags);
365 /* Remove the device from the domain device list */
366 list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
367 if (!dev || (info->dev == dev))
368 remove_device_ref(info, dma_domain->win_cnt);
369 }
370 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
371 }
372
373 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
374 {
375 struct device_domain_info *info, *old_domain_info;
376 unsigned long flags;
377
378 spin_lock_irqsave(&device_domain_lock, flags);
379 /*
380 * Check here if the device is already attached to domain or not.
381 * If the device is already attached to a domain detach it.
382 */
383 old_domain_info = find_domain(dev);
384 if (old_domain_info && old_domain_info->domain != dma_domain) {
385 spin_unlock_irqrestore(&device_domain_lock, flags);
386 detach_device(dev, old_domain_info->domain);
387 spin_lock_irqsave(&device_domain_lock, flags);
388 }
389
390 info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
391
392 info->dev = dev;
393 info->liodn = liodn;
394 info->domain = dma_domain;
395
396 list_add(&info->link, &dma_domain->devices);
397 /*
398 * In case of devices with multiple LIODNs just store
399 * the info for the first LIODN as all
400 * LIODNs share the same domain
401 */
402 if (!old_domain_info)
403 dev->archdata.iommu_domain = info;
404 spin_unlock_irqrestore(&device_domain_lock, flags);
405
406 }
407
408 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
409 dma_addr_t iova)
410 {
411 struct fsl_dma_domain *dma_domain = domain->priv;
412
413 if ((iova < domain->geometry.aperture_start) ||
414 iova > (domain->geometry.aperture_end))
415 return 0;
416
417 return get_phys_addr(dma_domain, iova);
418 }
419
420 static int fsl_pamu_domain_has_cap(struct iommu_domain *domain,
421 unsigned long cap)
422 {
423 return cap == IOMMU_CAP_CACHE_COHERENCY;
424 }
425
426 static void fsl_pamu_domain_destroy(struct iommu_domain *domain)
427 {
428 struct fsl_dma_domain *dma_domain = domain->priv;
429
430 domain->priv = NULL;
431
432 /* remove all the devices from the device list */
433 detach_device(NULL, dma_domain);
434
435 dma_domain->enabled = 0;
436 dma_domain->mapped = 0;
437
438 kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
439 }
440
441 static int fsl_pamu_domain_init(struct iommu_domain *domain)
442 {
443 struct fsl_dma_domain *dma_domain;
444
445 dma_domain = iommu_alloc_dma_domain();
446 if (!dma_domain) {
447 pr_debug("dma_domain allocation failed\n");
448 return -ENOMEM;
449 }
450 domain->priv = dma_domain;
451 dma_domain->iommu_domain = domain;
452 /* defaul geometry 64 GB i.e. maximum system address */
453 domain->geometry.aperture_start = 0;
454 domain->geometry.aperture_end = (1ULL << 36) - 1;
455 domain->geometry.force_aperture = true;
456
457 return 0;
458 }
459
460 /* Configure geometry settings for all LIODNs associated with domain */
461 static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain,
462 struct iommu_domain_geometry *geom_attr,
463 u32 win_cnt)
464 {
465 struct device_domain_info *info;
466 int ret = 0;
467
468 list_for_each_entry(info, &dma_domain->devices, link) {
469 ret = pamu_set_liodn(info->liodn, info->dev, dma_domain,
470 geom_attr, win_cnt);
471 if (ret)
472 break;
473 }
474
475 return ret;
476 }
477
478 /* Update stash destination for all LIODNs associated with the domain */
479 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
480 {
481 struct device_domain_info *info;
482 int ret = 0;
483
484 list_for_each_entry(info, &dma_domain->devices, link) {
485 ret = update_liodn_stash(info->liodn, dma_domain, val);
486 if (ret)
487 break;
488 }
489
490 return ret;
491 }
492
493 /* Update domain mappings for all LIODNs associated with the domain */
494 static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
495 {
496 struct device_domain_info *info;
497 int ret = 0;
498
499 list_for_each_entry(info, &dma_domain->devices, link) {
500 ret = update_liodn(info->liodn, dma_domain, wnd_nr);
501 if (ret)
502 break;
503 }
504 return ret;
505 }
506
507 static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
508 {
509 struct device_domain_info *info;
510 int ret = 0;
511
512 list_for_each_entry(info, &dma_domain->devices, link) {
513 if (dma_domain->win_cnt == 1 && dma_domain->enabled) {
514 ret = pamu_disable_liodn(info->liodn);
515 if (!ret)
516 dma_domain->enabled = 0;
517 } else {
518 ret = pamu_disable_spaace(info->liodn, wnd_nr);
519 }
520 }
521
522 return ret;
523 }
524
525 static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
526 {
527 struct fsl_dma_domain *dma_domain = domain->priv;
528 unsigned long flags;
529 int ret;
530
531 spin_lock_irqsave(&dma_domain->domain_lock, flags);
532 if (!dma_domain->win_arr) {
533 pr_debug("Number of windows not configured\n");
534 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
535 return;
536 }
537
538 if (wnd_nr >= dma_domain->win_cnt) {
539 pr_debug("Invalid window index\n");
540 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
541 return;
542 }
543
544 if (dma_domain->win_arr[wnd_nr].valid) {
545 ret = disable_domain_win(dma_domain, wnd_nr);
546 if (!ret) {
547 dma_domain->win_arr[wnd_nr].valid = 0;
548 dma_domain->mapped--;
549 }
550 }
551
552 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
553
554 }
555
556 static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr,
557 phys_addr_t paddr, u64 size, int prot)
558 {
559 struct fsl_dma_domain *dma_domain = domain->priv;
560 struct dma_window *wnd;
561 int pamu_prot = 0;
562 int ret;
563 unsigned long flags;
564 u64 win_size;
565
566 if (prot & IOMMU_READ)
567 pamu_prot |= PAACE_AP_PERMS_QUERY;
568 if (prot & IOMMU_WRITE)
569 pamu_prot |= PAACE_AP_PERMS_UPDATE;
570
571 spin_lock_irqsave(&dma_domain->domain_lock, flags);
572 if (!dma_domain->win_arr) {
573 pr_debug("Number of windows not configured\n");
574 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
575 return -ENODEV;
576 }
577
578 if (wnd_nr >= dma_domain->win_cnt) {
579 pr_debug("Invalid window index\n");
580 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
581 return -EINVAL;
582 }
583
584 win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt);
585 if (size > win_size) {
586 pr_debug("Invalid window size \n");
587 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
588 return -EINVAL;
589 }
590
591 if (dma_domain->win_cnt == 1) {
592 if (dma_domain->enabled) {
593 pr_debug("Disable the window before updating the mapping\n");
594 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
595 return -EBUSY;
596 }
597
598 ret = check_size(size, domain->geometry.aperture_start);
599 if (ret) {
600 pr_debug("Aperture start not aligned to the size\n");
601 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
602 return -EINVAL;
603 }
604 }
605
606 wnd = &dma_domain->win_arr[wnd_nr];
607 if (!wnd->valid) {
608 wnd->paddr = paddr;
609 wnd->size = size;
610 wnd->prot = pamu_prot;
611
612 ret = update_domain_mapping(dma_domain, wnd_nr);
613 if (!ret) {
614 wnd->valid = 1;
615 dma_domain->mapped++;
616 }
617 } else {
618 pr_debug("Disable the window before updating the mapping\n");
619 ret = -EBUSY;
620 }
621
622 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
623
624 return ret;
625 }
626
627 /*
628 * Attach the LIODN to the DMA domain and configure the geometry
629 * and window mappings.
630 */
631 static int handle_attach_device(struct fsl_dma_domain *dma_domain,
632 struct device *dev, const u32 *liodn,
633 int num)
634 {
635 unsigned long flags;
636 struct iommu_domain *domain = dma_domain->iommu_domain;
637 int ret = 0;
638 int i;
639
640 spin_lock_irqsave(&dma_domain->domain_lock, flags);
641 for (i = 0; i < num; i++) {
642
643 /* Ensure that LIODN value is valid */
644 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
645 pr_debug("Invalid liodn %d, attach device failed for %s\n",
646 liodn[i], dev->of_node->full_name);
647 ret = -EINVAL;
648 break;
649 }
650
651 attach_device(dma_domain, liodn[i], dev);
652 /*
653 * Check if geometry has already been configured
654 * for the domain. If yes, set the geometry for
655 * the LIODN.
656 */
657 if (dma_domain->win_arr) {
658 u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0;
659 ret = pamu_set_liodn(liodn[i], dev, dma_domain,
660 &domain->geometry,
661 win_cnt);
662 if (ret)
663 break;
664 if (dma_domain->mapped) {
665 /*
666 * Create window/subwindow mapping for
667 * the LIODN.
668 */
669 ret = map_liodn(liodn[i], dma_domain);
670 if (ret)
671 break;
672 }
673 }
674 }
675 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
676
677 return ret;
678 }
679
680 static int fsl_pamu_attach_device(struct iommu_domain *domain,
681 struct device *dev)
682 {
683 struct fsl_dma_domain *dma_domain = domain->priv;
684 const u32 *liodn;
685 u32 liodn_cnt;
686 int len, ret = 0;
687 struct pci_dev *pdev = NULL;
688 struct pci_controller *pci_ctl;
689
690 /*
691 * Use LIODN of the PCI controller while attaching a
692 * PCI device.
693 */
694 if (dev->bus == &pci_bus_type) {
695 pdev = to_pci_dev(dev);
696 pci_ctl = pci_bus_to_host(pdev->bus);
697 /*
698 * make dev point to pci controller device
699 * so we can get the LIODN programmed by
700 * u-boot.
701 */
702 dev = pci_ctl->parent;
703 }
704
705 liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
706 if (liodn) {
707 liodn_cnt = len / sizeof(u32);
708 ret = handle_attach_device(dma_domain, dev,
709 liodn, liodn_cnt);
710 } else {
711 pr_debug("missing fsl,liodn property at %s\n",
712 dev->of_node->full_name);
713 ret = -EINVAL;
714 }
715
716 return ret;
717 }
718
719 static void fsl_pamu_detach_device(struct iommu_domain *domain,
720 struct device *dev)
721 {
722 struct fsl_dma_domain *dma_domain = domain->priv;
723 const u32 *prop;
724 int len;
725 struct pci_dev *pdev = NULL;
726 struct pci_controller *pci_ctl;
727
728 /*
729 * Use LIODN of the PCI controller while detaching a
730 * PCI device.
731 */
732 if (dev->bus == &pci_bus_type) {
733 pdev = to_pci_dev(dev);
734 pci_ctl = pci_bus_to_host(pdev->bus);
735 /*
736 * make dev point to pci controller device
737 * so we can get the LIODN programmed by
738 * u-boot.
739 */
740 dev = pci_ctl->parent;
741 }
742
743 prop = of_get_property(dev->of_node, "fsl,liodn", &len);
744 if (prop)
745 detach_device(dev, dma_domain);
746 else
747 pr_debug("missing fsl,liodn property at %s\n",
748 dev->of_node->full_name);
749 }
750
751 static int configure_domain_geometry(struct iommu_domain *domain, void *data)
752 {
753 struct iommu_domain_geometry *geom_attr = data;
754 struct fsl_dma_domain *dma_domain = domain->priv;
755 dma_addr_t geom_size;
756 unsigned long flags;
757
758 geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1;
759 /*
760 * Sanity check the geometry size. Also, we do not support
761 * DMA outside of the geometry.
762 */
763 if (check_size(geom_size, geom_attr->aperture_start) ||
764 !geom_attr->force_aperture) {
765 pr_debug("Invalid PAMU geometry attributes\n");
766 return -EINVAL;
767 }
768
769 spin_lock_irqsave(&dma_domain->domain_lock, flags);
770 if (dma_domain->enabled) {
771 pr_debug("Can't set geometry attributes as domain is active\n");
772 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
773 return -EBUSY;
774 }
775
776 /* Copy the domain geometry information */
777 memcpy(&domain->geometry, geom_attr,
778 sizeof(struct iommu_domain_geometry));
779 dma_domain->geom_size = geom_size;
780
781 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
782
783 return 0;
784 }
785
786 /* Set the domain stash attribute */
787 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)
788 {
789 struct pamu_stash_attribute *stash_attr = data;
790 unsigned long flags;
791 int ret;
792
793 spin_lock_irqsave(&dma_domain->domain_lock, flags);
794
795 memcpy(&dma_domain->dma_stash, stash_attr,
796 sizeof(struct pamu_stash_attribute));
797
798 dma_domain->stash_id = get_stash_id(stash_attr->cache,
799 stash_attr->cpu);
800 if (dma_domain->stash_id == ~(u32)0) {
801 pr_debug("Invalid stash attributes\n");
802 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
803 return -EINVAL;
804 }
805
806 ret = update_domain_stash(dma_domain, dma_domain->stash_id);
807
808 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
809
810 return ret;
811 }
812
813 /* Configure domain dma state i.e. enable/disable DMA*/
814 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)
815 {
816 struct device_domain_info *info;
817 unsigned long flags;
818 int ret;
819
820 spin_lock_irqsave(&dma_domain->domain_lock, flags);
821
822 if (enable && !dma_domain->mapped) {
823 pr_debug("Can't enable DMA domain without valid mapping\n");
824 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
825 return -ENODEV;
826 }
827
828 dma_domain->enabled = enable;
829 list_for_each_entry(info, &dma_domain->devices,
830 link) {
831 ret = (enable) ? pamu_enable_liodn(info->liodn) :
832 pamu_disable_liodn(info->liodn);
833 if (ret)
834 pr_debug("Unable to set dma state for liodn %d",
835 info->liodn);
836 }
837 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
838
839 return 0;
840 }
841
842 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
843 enum iommu_attr attr_type, void *data)
844 {
845 struct fsl_dma_domain *dma_domain = domain->priv;
846 int ret = 0;
847
848
849 switch (attr_type) {
850 case DOMAIN_ATTR_GEOMETRY:
851 ret = configure_domain_geometry(domain, data);
852 break;
853 case DOMAIN_ATTR_FSL_PAMU_STASH:
854 ret = configure_domain_stash(dma_domain, data);
855 break;
856 case DOMAIN_ATTR_FSL_PAMU_ENABLE:
857 ret = configure_domain_dma_state(dma_domain, *(int *)data);
858 break;
859 default:
860 pr_debug("Unsupported attribute type\n");
861 ret = -EINVAL;
862 break;
863 };
864
865 return ret;
866 }
867
868 static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
869 enum iommu_attr attr_type, void *data)
870 {
871 struct fsl_dma_domain *dma_domain = domain->priv;
872 int ret = 0;
873
874
875 switch (attr_type) {
876 case DOMAIN_ATTR_FSL_PAMU_STASH:
877 memcpy((struct pamu_stash_attribute *) data, &dma_domain->dma_stash,
878 sizeof(struct pamu_stash_attribute));
879 break;
880 case DOMAIN_ATTR_FSL_PAMU_ENABLE:
881 *(int *)data = dma_domain->enabled;
882 break;
883 case DOMAIN_ATTR_FSL_PAMUV1:
884 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1;
885 break;
886 default:
887 pr_debug("Unsupported attribute type\n");
888 ret = -EINVAL;
889 break;
890 };
891
892 return ret;
893 }
894
895 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
896
897 static struct iommu_group *get_device_iommu_group(struct device *dev)
898 {
899 struct iommu_group *group;
900
901 group = iommu_group_get(dev);
902 if (!group)
903 group = iommu_group_alloc();
904
905 return group;
906 }
907
908 static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
909 {
910 u32 version;
911
912 /* Check the PCI controller version number by readding BRR1 register */
913 version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
914 version &= PCI_FSL_BRR1_VER;
915 /* If PCI controller version is >= 0x204 we can partition endpoints*/
916 if (version >= 0x204)
917 return 1;
918
919 return 0;
920 }
921
922 /* Get iommu group information from peer devices or devices on the parent bus */
923 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
924 {
925 struct pci_dev *tmp;
926 struct iommu_group *group;
927 struct pci_bus *bus = pdev->bus;
928
929 /*
930 * Traverese the pci bus device list to get
931 * the shared iommu group.
932 */
933 while (bus) {
934 list_for_each_entry(tmp, &bus->devices, bus_list) {
935 if (tmp == pdev)
936 continue;
937 group = iommu_group_get(&tmp->dev);
938 if (group)
939 return group;
940 }
941
942 bus = bus->parent;
943 }
944
945 return NULL;
946 }
947
948 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
949 {
950 struct pci_controller *pci_ctl;
951 bool pci_endpt_partioning;
952 struct iommu_group *group = NULL;
953 struct pci_dev *bridge, *dma_pdev = NULL;
954
955 pci_ctl = pci_bus_to_host(pdev->bus);
956 pci_endpt_partioning = check_pci_ctl_endpt_part(pci_ctl);
957 /* We can partition PCIe devices so assign device group to the device */
958 if (pci_endpt_partioning) {
959 bridge = pci_find_upstream_pcie_bridge(pdev);
960 if (bridge) {
961 if (pci_is_pcie(bridge))
962 dma_pdev = pci_get_domain_bus_and_slot(
963 pci_domain_nr(pdev->bus),
964 bridge->subordinate->number, 0);
965 if (!dma_pdev)
966 dma_pdev = pci_dev_get(bridge);
967 } else
968 dma_pdev = pci_dev_get(pdev);
969
970 /* Account for quirked devices */
971 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
972
973 /*
974 * If it's a multifunction device that does not support our
975 * required ACS flags, add to the same group as lowest numbered
976 * function that also does not suport the required ACS flags.
977 */
978 if (dma_pdev->multifunction &&
979 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
980 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
981
982 for (i = 0; i < 8; i++) {
983 struct pci_dev *tmp;
984
985 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
986 if (!tmp)
987 continue;
988
989 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
990 swap_pci_ref(&dma_pdev, tmp);
991 break;
992 }
993 pci_dev_put(tmp);
994 }
995 }
996
997 /*
998 * Devices on the root bus go through the iommu. If that's not us,
999 * find the next upstream device and test ACS up to the root bus.
1000 * Finding the next device may require skipping virtual buses.
1001 */
1002 while (!pci_is_root_bus(dma_pdev->bus)) {
1003 struct pci_bus *bus = dma_pdev->bus;
1004
1005 while (!bus->self) {
1006 if (!pci_is_root_bus(bus))
1007 bus = bus->parent;
1008 else
1009 goto root_bus;
1010 }
1011
1012 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1013 break;
1014
1015 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
1016 }
1017
1018 root_bus:
1019 group = get_device_iommu_group(&dma_pdev->dev);
1020 pci_dev_put(dma_pdev);
1021 /*
1022 * PCIe controller is not a paritionable entity
1023 * free the controller device iommu_group.
1024 */
1025 if (pci_ctl->parent->iommu_group)
1026 iommu_group_remove_device(pci_ctl->parent);
1027 } else {
1028 /*
1029 * All devices connected to the controller will share the
1030 * PCI controllers device group. If this is the first
1031 * device to be probed for the pci controller, copy the
1032 * device group information from the PCI controller device
1033 * node and remove the PCI controller iommu group.
1034 * For subsequent devices, the iommu group information can
1035 * be obtained from sibling devices (i.e. from the bus_devices
1036 * link list).
1037 */
1038 if (pci_ctl->parent->iommu_group) {
1039 group = get_device_iommu_group(pci_ctl->parent);
1040 iommu_group_remove_device(pci_ctl->parent);
1041 } else
1042 group = get_shared_pci_device_group(pdev);
1043 }
1044
1045 return group;
1046 }
1047
1048 static int fsl_pamu_add_device(struct device *dev)
1049 {
1050 struct iommu_group *group = NULL;
1051 struct pci_dev *pdev;
1052 const u32 *prop;
1053 int ret, len;
1054
1055 /*
1056 * For platform devices we allocate a separate group for
1057 * each of the devices.
1058 */
1059 if (dev->bus == &pci_bus_type) {
1060 pdev = to_pci_dev(dev);
1061 /* Don't create device groups for virtual PCI bridges */
1062 if (pdev->subordinate)
1063 return 0;
1064
1065 group = get_pci_device_group(pdev);
1066
1067 } else {
1068 prop = of_get_property(dev->of_node, "fsl,liodn", &len);
1069 if (prop)
1070 group = get_device_iommu_group(dev);
1071 }
1072
1073 if (!group || IS_ERR(group))
1074 return PTR_ERR(group);
1075
1076 ret = iommu_group_add_device(group, dev);
1077
1078 iommu_group_put(group);
1079 return ret;
1080 }
1081
1082 static void fsl_pamu_remove_device(struct device *dev)
1083 {
1084 iommu_group_remove_device(dev);
1085 }
1086
1087 static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
1088 {
1089 struct fsl_dma_domain *dma_domain = domain->priv;
1090 unsigned long flags;
1091 int ret;
1092
1093 spin_lock_irqsave(&dma_domain->domain_lock, flags);
1094 /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
1095 if (dma_domain->enabled) {
1096 pr_debug("Can't set geometry attributes as domain is active\n");
1097 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1098 return -EBUSY;
1099 }
1100
1101 /* Ensure that the geometry has been set for the domain */
1102 if (!dma_domain->geom_size) {
1103 pr_debug("Please configure geometry before setting the number of windows\n");
1104 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1105 return -EINVAL;
1106 }
1107
1108 /*
1109 * Ensure we have valid window count i.e. it should be less than
1110 * maximum permissible limit and should be a power of two.
1111 */
1112 if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
1113 pr_debug("Invalid window count\n");
1114 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1115 return -EINVAL;
1116 }
1117
1118 ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
1119 ((w_count > 1) ? w_count : 0));
1120 if (!ret) {
1121 if (dma_domain->win_arr)
1122 kfree(dma_domain->win_arr);
1123 dma_domain->win_arr = kzalloc(sizeof(struct dma_window) *
1124 w_count, GFP_ATOMIC);
1125 if (!dma_domain->win_arr) {
1126 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1127 return -ENOMEM;
1128 }
1129 dma_domain->win_cnt = w_count;
1130 }
1131 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1132
1133 return ret;
1134 }
1135
1136 static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
1137 {
1138 struct fsl_dma_domain *dma_domain = domain->priv;
1139
1140 return dma_domain->win_cnt;
1141 }
1142
1143 static struct iommu_ops fsl_pamu_ops = {
1144 .domain_init = fsl_pamu_domain_init,
1145 .domain_destroy = fsl_pamu_domain_destroy,
1146 .attach_dev = fsl_pamu_attach_device,
1147 .detach_dev = fsl_pamu_detach_device,
1148 .domain_window_enable = fsl_pamu_window_enable,
1149 .domain_window_disable = fsl_pamu_window_disable,
1150 .domain_get_windows = fsl_pamu_get_windows,
1151 .domain_set_windows = fsl_pamu_set_windows,
1152 .iova_to_phys = fsl_pamu_iova_to_phys,
1153 .domain_has_cap = fsl_pamu_domain_has_cap,
1154 .domain_set_attr = fsl_pamu_set_domain_attr,
1155 .domain_get_attr = fsl_pamu_get_domain_attr,
1156 .add_device = fsl_pamu_add_device,
1157 .remove_device = fsl_pamu_remove_device,
1158 };
1159
1160 int pamu_domain_init()
1161 {
1162 int ret = 0;
1163
1164 ret = iommu_init_mempool();
1165 if (ret)
1166 return ret;
1167
1168 bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
1169 bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
1170
1171 return ret;
1172 }
Linus' kernel tree