2 * Copyright 2012 Tilera Corporation. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
15 #include <linux/kernel.h>
16 #include <linux/mmzone.h>
17 #include <linux/pci.h>
18 #include <linux/delay.h>
19 #include <linux/string.h>
20 #include <linux/init.h>
21 #include <linux/capability.h>
22 #include <linux/sched.h>
23 #include <linux/errno.h>
24 #include <linux/irq.h>
25 #include <linux/msi.h>
27 #include <linux/uaccess.h>
28 #include <linux/ctype.h>
30 #include <asm/processor.h>
31 #include <asm/sections.h>
32 #include <asm/byteorder.h>
34 #include <gxio/iorpc_globals.h>
35 #include <gxio/kiorpc.h>
36 #include <gxio/trio.h>
37 #include <gxio/iorpc_trio.h>
38 #include <hv/drv_trio_intf.h>
43 * This file containes the routines to search for PCI buses,
44 * enumerate the buses, and configure any attached devices.
47 #define DEBUG_PCI_CFG 0
50 #define TRACE_CFG_WR(size, val, bus, dev, func, offset) \
51 pr_info("CFG WR %d-byte VAL %#x to bus %d dev %d func %d addr %u\n", \
52 size, val, bus, dev, func, offset & 0xFFF);
53 #define TRACE_CFG_RD(size, val, bus, dev, func, offset) \
54 pr_info("CFG RD %d-byte VAL %#x from bus %d dev %d func %d addr %u\n", \
55 size, val, bus, dev, func, offset & 0xFFF);
57 #define TRACE_CFG_WR(...)
58 #define TRACE_CFG_RD(...)
61 static int pci_probe
= 1;
63 /* Information on the PCIe RC ports configuration. */
64 static int pcie_rc
[TILEGX_NUM_TRIO
][TILEGX_TRIO_PCIES
];
67 * On some platforms with one or more Gx endpoint ports, we need to
68 * delay the PCIe RC port probe for a few seconds to work around
69 * a HW PCIe link-training bug. The exact delay is specified with
70 * a kernel boot argument in the form of "pcie_rc_delay=T,P,S",
71 * where T is the TRIO instance number, P is the port number and S is
72 * the delay in seconds. If the argument is specified, but the delay is
73 * not provided, the value will be DEFAULT_RC_DELAY.
75 static int rc_delay
[TILEGX_NUM_TRIO
][TILEGX_TRIO_PCIES
];
77 /* Default number of seconds that the PCIe RC port probe can be delayed. */
78 #define DEFAULT_RC_DELAY 10
80 /* Array of the PCIe ports configuration info obtained from the BIB. */
81 struct pcie_port_property pcie_ports
[TILEGX_NUM_TRIO
][TILEGX_TRIO_PCIES
];
83 /* All drivers share the TRIO contexts defined here. */
84 gxio_trio_context_t trio_contexts
[TILEGX_NUM_TRIO
];
86 /* Pointer to an array of PCIe RC controllers. */
87 struct pci_controller pci_controllers
[TILEGX_NUM_TRIO
* TILEGX_TRIO_PCIES
];
88 int num_rc_controllers
;
89 static int num_ep_controllers
;
91 static struct pci_ops tile_cfg_ops
;
93 /* Mask of CPUs that should receive PCIe interrupts. */
94 static struct cpumask intr_cpus_map
;
97 * We don't need to worry about the alignment of resources.
99 resource_size_t
pcibios_align_resource(void *data
, const struct resource
*res
,
100 resource_size_t size
, resource_size_t align
)
104 EXPORT_SYMBOL(pcibios_align_resource
);
108 * Pick a CPU to receive and handle the PCIe interrupts, based on the IRQ #.
109 * For now, we simply send interrupts to non-dataplane CPUs.
110 * We may implement methods to allow user to specify the target CPUs,
111 * e.g. via boot arguments.
113 static int tile_irq_cpu(int irq
)
119 count
= cpumask_weight(&intr_cpus_map
);
120 if (unlikely(count
== 0)) {
121 pr_warning("intr_cpus_map empty, interrupts will be"
122 " delievered to dataplane tiles\n");
123 return irq
% (smp_height
* smp_width
);
127 for_each_cpu(cpu
, &intr_cpus_map
) {
135 * Open a file descriptor to the TRIO shim.
137 static int tile_pcie_open(int trio_index
)
139 gxio_trio_context_t
*context
= &trio_contexts
[trio_index
];
143 * This opens a file descriptor to the TRIO shim.
145 ret
= gxio_trio_init(context
, trio_index
);
150 * Allocate an ASID for the kernel.
152 ret
= gxio_trio_alloc_asids(context
, 1, 0, 0);
154 pr_err("PCI: ASID alloc failure on TRIO %d, give up\n",
156 goto asid_alloc_failure
;
161 #ifdef USE_SHARED_PCIE_CONFIG_REGION
163 * Alloc a PIO region for config access, shared by all MACs per TRIO.
164 * This shouldn't fail since the kernel is supposed to the first
165 * client of the TRIO's PIO regions.
167 ret
= gxio_trio_alloc_pio_regions(context
, 1, 0, 0);
169 pr_err("PCI: CFG PIO alloc failure on TRIO %d, give up\n",
171 goto pio_alloc_failure
;
174 context
->pio_cfg_index
= ret
;
177 * For PIO CFG, the bus_address_hi parameter is 0. The mac parameter
178 * is also 0 because it is specified in PIO_REGION_SETUP_CFG_ADDR.
180 ret
= gxio_trio_init_pio_region_aux(context
, context
->pio_cfg_index
,
181 0, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE
);
183 pr_err("PCI: CFG PIO init failure on TRIO %d, give up\n",
185 goto pio_alloc_failure
;
192 #ifdef USE_SHARED_PCIE_CONFIG_REGION
195 hv_dev_close(context
->fd
);
201 tilegx_legacy_irq_ack(struct irq_data
*d
)
203 __insn_mtspr(SPR_IPI_EVENT_RESET_K
, 1UL << d
->irq
);
207 tilegx_legacy_irq_mask(struct irq_data
*d
)
209 __insn_mtspr(SPR_IPI_MASK_SET_K
, 1UL << d
->irq
);
213 tilegx_legacy_irq_unmask(struct irq_data
*d
)
215 __insn_mtspr(SPR_IPI_MASK_RESET_K
, 1UL << d
->irq
);
218 static struct irq_chip tilegx_legacy_irq_chip
= {
219 .name
= "tilegx_legacy_irq",
220 .irq_ack
= tilegx_legacy_irq_ack
,
221 .irq_mask
= tilegx_legacy_irq_mask
,
222 .irq_unmask
= tilegx_legacy_irq_unmask
,
224 /* TBD: support set_affinity. */
228 * This is a wrapper function of the kernel level-trigger interrupt
229 * handler handle_level_irq() for PCI legacy interrupts. The TRIO
230 * is configured such that only INTx Assert interrupts are proxied
231 * to Linux which just calls handle_level_irq() after clearing the
232 * MAC INTx Assert status bit associated with this interrupt.
235 trio_handle_level_irq(unsigned int irq
, struct irq_desc
*desc
)
237 struct pci_controller
*controller
= irq_desc_get_handler_data(desc
);
238 gxio_trio_context_t
*trio_context
= controller
->trio
;
239 uint64_t intx
= (uint64_t)irq_desc_get_chip_data(desc
);
240 int mac
= controller
->mac
;
241 unsigned int reg_offset
;
244 handle_level_irq(irq
, desc
);
247 * Clear the INTx Level status, otherwise future interrupts are
250 reg_offset
= (TRIO_PCIE_INTFC_MAC_INT_STS
<<
251 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
252 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE
<<
253 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
254 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
256 level_mask
= TRIO_PCIE_INTFC_MAC_INT_STS__INT_LEVEL_MASK
<< intx
;
258 __gxio_mmio_write(trio_context
->mmio_base_mac
+ reg_offset
, level_mask
);
262 * Create kernel irqs and set up the handlers for the legacy interrupts.
263 * Also some minimum initialization for the MSI support.
265 static int tile_init_irqs(struct pci_controller
*controller
)
272 cpumask_copy(&intr_cpus_map
, cpu_online_mask
);
275 for (i
= 0; i
< 4; i
++) {
276 gxio_trio_context_t
*context
= controller
->trio
;
279 /* Ask the kernel to allocate an IRQ. */
282 pr_err("PCI: no free irq vectors, failed for %d\n", i
);
286 controller
->irq_intx_table
[i
] = irq
;
288 /* Distribute the 4 IRQs to different tiles. */
289 cpu
= tile_irq_cpu(irq
);
291 /* Configure the TRIO intr binding for this IRQ. */
292 result
= gxio_trio_config_legacy_intr(context
, cpu_x(cpu
),
293 cpu_y(cpu
), KERNEL_PL
,
294 irq
, controller
->mac
, i
);
296 pr_err("PCI: MAC intx config failed for %d\n", i
);
302 * Register the IRQ handler with the kernel.
304 irq_set_chip_and_handler(irq
, &tilegx_legacy_irq_chip
,
305 trio_handle_level_irq
);
306 irq_set_chip_data(irq
, (void *)(uint64_t)i
);
307 irq_set_handler_data(irq
, controller
);
313 for (j
= 0; j
< i
; j
++)
314 destroy_irq(controller
->irq_intx_table
[j
]);
320 * Find valid controllers and fill in pci_controller structs for each
323 * Returns the number of controllers discovered.
325 int __init
tile_pci_init(void)
327 int num_trio_shims
= 0;
332 pr_info("PCI: disabled by boot argument\n");
336 pr_info("PCI: Searching for controllers...\n");
339 * We loop over all the TRIO shims.
341 for (i
= 0; i
< TILEGX_NUM_TRIO
; i
++) {
344 ret
= tile_pcie_open(i
);
351 if (num_trio_shims
== 0 || sim_is_simulator())
355 * Now determine which PCIe ports are configured to operate in RC mode.
356 * We look at the Board Information Block first and then see if there
357 * are any overriding configuration by the HW strapping pin.
359 for (i
= 0; i
< TILEGX_NUM_TRIO
; i
++) {
360 gxio_trio_context_t
*context
= &trio_contexts
[i
];
366 ret
= hv_dev_pread(context
->fd
, 0,
367 (HV_VirtAddr
)&pcie_ports
[i
][0],
368 sizeof(struct pcie_port_property
) * TILEGX_TRIO_PCIES
,
369 GXIO_TRIO_OP_GET_PORT_PROPERTY
);
371 pr_err("PCI: PCIE_GET_PORT_PROPERTY failure, error %d,"
372 " on TRIO %d\n", ret
, i
);
376 for (j
= 0; j
< TILEGX_TRIO_PCIES
; j
++) {
377 if (pcie_ports
[i
][j
].allow_rc
) {
379 num_rc_controllers
++;
381 else if (pcie_ports
[i
][j
].allow_ep
) {
382 num_ep_controllers
++;
388 * Return if no PCIe ports are configured to operate in RC mode.
390 if (num_rc_controllers
== 0)
394 * Set the TRIO pointer and MAC index for each PCIe RC port.
396 for (i
= 0; i
< TILEGX_NUM_TRIO
; i
++) {
397 for (j
= 0; j
< TILEGX_TRIO_PCIES
; j
++) {
399 pci_controllers
[ctl_index
].trio
=
401 pci_controllers
[ctl_index
].mac
= j
;
402 pci_controllers
[ctl_index
].trio_index
= i
;
404 if (ctl_index
== num_rc_controllers
)
412 * Configure each PCIe RC port.
414 for (i
= 0; i
< num_rc_controllers
; i
++) {
416 * Configure the PCIe MAC to run in RC mode.
419 struct pci_controller
*controller
= &pci_controllers
[i
];
421 controller
->index
= i
;
422 controller
->ops
= &tile_cfg_ops
;
425 * The PCI memory resource is located above the PA space.
426 * For every host bridge, the BAR window or the MMIO aperture
427 * is in range [3GB, 4GB - 1] of a 4GB space beyond the
431 controller
->mem_offset
= TILE_PCI_MEM_START
+
432 (i
* TILE_PCI_BAR_WINDOW_TOP
);
433 controller
->mem_space
.start
= controller
->mem_offset
+
434 TILE_PCI_BAR_WINDOW_TOP
- TILE_PCI_BAR_WINDOW_SIZE
;
435 controller
->mem_space
.end
= controller
->mem_offset
+
436 TILE_PCI_BAR_WINDOW_TOP
- 1;
437 controller
->mem_space
.flags
= IORESOURCE_MEM
;
438 snprintf(controller
->mem_space_name
,
439 sizeof(controller
->mem_space_name
),
440 "PCI mem domain %d", i
);
441 controller
->mem_space
.name
= controller
->mem_space_name
;
444 return num_rc_controllers
;
448 * (pin - 1) converts from the PCI standard's [1:4] convention to
449 * a normal [0:3] range.
451 static int tile_map_irq(const struct pci_dev
*dev
, u8 device
, u8 pin
)
453 struct pci_controller
*controller
=
454 (struct pci_controller
*)dev
->sysdata
;
455 return controller
->irq_intx_table
[pin
- 1];
459 static void fixup_read_and_payload_sizes(struct pci_controller
*controller
)
461 gxio_trio_context_t
*trio_context
= controller
->trio
;
462 struct pci_bus
*root_bus
= controller
->root_bus
;
463 TRIO_PCIE_RC_DEVICE_CONTROL_t dev_control
;
464 TRIO_PCIE_RC_DEVICE_CAP_t rc_dev_cap
;
465 unsigned int reg_offset
;
466 struct pci_bus
*child
;
470 mac
= controller
->mac
;
473 * Set our max read request size to be 4KB.
476 (TRIO_PCIE_RC_DEVICE_CONTROL
<<
477 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
478 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD
<<
479 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
480 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
482 dev_control
.word
= __gxio_mmio_read32(trio_context
->mmio_base_mac
+
484 dev_control
.max_read_req_sz
= 5;
485 __gxio_mmio_write32(trio_context
->mmio_base_mac
+ reg_offset
,
489 * Set the max payload size supported by this Gx PCIe MAC.
490 * Though Gx PCIe supports Max Payload Size of up to 1024 bytes,
491 * experiments have shown that setting MPS to 256 yields the
495 (TRIO_PCIE_RC_DEVICE_CAP
<<
496 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
497 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD
<<
498 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
499 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
501 rc_dev_cap
.word
= __gxio_mmio_read32(trio_context
->mmio_base_mac
+
503 rc_dev_cap
.mps_sup
= 1;
504 __gxio_mmio_write32(trio_context
->mmio_base_mac
+ reg_offset
,
507 /* Configure PCI Express MPS setting. */
508 list_for_each_entry(child
, &root_bus
->children
, node
) {
509 struct pci_dev
*self
= child
->self
;
513 pcie_bus_configure_settings(child
, self
->pcie_mpss
);
517 * Set the mac_config register in trio based on the MPS/MRS of the link.
520 (TRIO_PCIE_RC_DEVICE_CONTROL
<<
521 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
522 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD
<<
523 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
524 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
526 dev_control
.word
= __gxio_mmio_read32(trio_context
->mmio_base_mac
+
529 err
= gxio_trio_set_mps_mrs(trio_context
,
530 dev_control
.max_payload_size
,
531 dev_control
.max_read_req_sz
,
534 pr_err("PCI: PCIE_CONFIGURE_MAC_MPS_MRS failure, "
535 "MAC %d on TRIO %d\n",
536 mac
, controller
->trio_index
);
540 static int setup_pcie_rc_delay(char *str
)
542 unsigned long delay
= 0;
543 unsigned long trio_index
;
546 if (str
== NULL
|| !isdigit(*str
))
548 trio_index
= simple_strtoul(str
, (char **)&str
, 10);
549 if (trio_index
>= TILEGX_NUM_TRIO
)
558 mac
= simple_strtoul(str
, (char **)&str
, 10);
559 if (mac
>= TILEGX_TRIO_PCIES
)
569 delay
= simple_strtoul(str
, (char **)&str
, 10);
572 rc_delay
[trio_index
][mac
] = delay
? : DEFAULT_RC_DELAY
;
575 early_param("pcie_rc_delay", setup_pcie_rc_delay
);
578 * PCI initialization entry point, called by subsys_initcall.
580 int __init
pcibios_init(void)
582 resource_size_t offset
;
583 LIST_HEAD(resources
);
589 if (num_rc_controllers
== 0 && num_ep_controllers
== 0)
593 * We loop over all the TRIO shims and set up the MMIO mappings.
595 for (i
= 0; i
< TILEGX_NUM_TRIO
; i
++) {
596 gxio_trio_context_t
*context
= &trio_contexts
[i
];
602 * Map in the MMIO space for the MAC.
605 context
->mmio_base_mac
=
606 iorpc_ioremap(context
->fd
, offset
,
607 HV_TRIO_CONFIG_IOREMAP_SIZE
);
608 if (context
->mmio_base_mac
== NULL
) {
609 pr_err("PCI: MAC map failure on TRIO %d\n", i
);
611 hv_dev_close(context
->fd
);
618 * Delay a bit in case devices aren't ready. Some devices are
619 * known to require at least 20ms here, but we use a more
620 * conservative value.
624 /* Scan all of the recorded PCI controllers. */
625 for (next_busno
= 0, i
= 0; i
< num_rc_controllers
; i
++) {
626 struct pci_controller
*controller
= &pci_controllers
[i
];
627 gxio_trio_context_t
*trio_context
= controller
->trio
;
628 TRIO_PCIE_INTFC_PORT_CONFIG_t port_config
;
629 TRIO_PCIE_INTFC_PORT_STATUS_t port_status
;
630 TRIO_PCIE_INTFC_TX_FIFO_CTL_t tx_fifo_ctl
;
632 unsigned int reg_offset
;
633 unsigned int class_code_revision
;
638 if (trio_context
->fd
< 0)
641 trio_index
= controller
->trio_index
;
642 mac
= controller
->mac
;
645 * Check the port strap state which will override the BIB
650 (TRIO_PCIE_INTFC_PORT_CONFIG
<<
651 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
652 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE
<<
653 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
654 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
657 __gxio_mmio_read(trio_context
->mmio_base_mac
+
660 if ((port_config
.strap_state
!=
661 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC
) &&
662 (port_config
.strap_state
!=
663 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC_G1
)) {
665 * If this is really intended to be an EP port,
666 * record it so that the endpoint driver will know about it.
668 if (port_config
.strap_state
==
669 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT
||
670 port_config
.strap_state
==
671 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT_G1
)
672 pcie_ports
[trio_index
][mac
].allow_ep
= 1;
678 * Check for PCIe link-up status to decide if we need
679 * to force the link to come up.
682 (TRIO_PCIE_INTFC_PORT_STATUS
<<
683 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
684 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE
<<
685 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
686 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
689 __gxio_mmio_read(trio_context
->mmio_base_mac
+
691 if (!port_status
.dl_up
) {
692 if (rc_delay
[trio_index
][mac
]) {
693 pr_info("Delaying PCIe RC TRIO init %d sec"
694 " on MAC %d on TRIO %d\n",
695 rc_delay
[trio_index
][mac
], mac
,
697 msleep(rc_delay
[trio_index
][mac
] * 1000);
699 ret
= gxio_trio_force_rc_link_up(trio_context
, mac
);
701 pr_err("PCI: PCIE_FORCE_LINK_UP failure, "
702 "MAC %d on TRIO %d\n", mac
, trio_index
);
705 pr_info("PCI: Found PCI controller #%d on TRIO %d MAC %d\n", i
,
706 trio_index
, controller
->mac
);
709 * Delay the bus probe if needed.
711 if (rc_delay
[trio_index
][mac
]) {
712 pr_info("Delaying PCIe RC bus enumerating %d sec"
713 " on MAC %d on TRIO %d\n",
714 rc_delay
[trio_index
][mac
], mac
,
716 msleep(rc_delay
[trio_index
][mac
] * 1000);
719 * Wait a bit here because some EP devices
720 * take longer to come up.
726 * Check for PCIe link-up status again.
729 __gxio_mmio_read(trio_context
->mmio_base_mac
+
731 if (!port_status
.dl_up
) {
732 if (pcie_ports
[trio_index
][mac
].removable
) {
733 pr_info("PCI: link is down, MAC %d on TRIO %d\n",
735 pr_info("This is expected if no PCIe card"
736 " is connected to this link\n");
738 pr_err("PCI: link is down, MAC %d on TRIO %d\n",
744 * Ensure that the link can come out of L1 power down state.
745 * Strictly speaking, this is needed only in the case of
746 * heavy RC-initiated DMAs.
749 (TRIO_PCIE_INTFC_TX_FIFO_CTL
<<
750 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
751 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE
<<
752 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
753 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
755 __gxio_mmio_read(trio_context
->mmio_base_mac
+
757 tx_fifo_ctl
.min_p_credits
= 0;
758 __gxio_mmio_write(trio_context
->mmio_base_mac
+ reg_offset
,
762 * Change the device ID so that Linux bus crawl doesn't confuse
763 * the internal bridge with any Tilera endpoints.
767 (TRIO_PCIE_RC_DEVICE_ID_VEN_ID
<<
768 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
769 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD
<<
770 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
771 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
773 __gxio_mmio_write32(trio_context
->mmio_base_mac
+ reg_offset
,
774 (TILERA_GX36_RC_DEV_ID
<<
775 TRIO_PCIE_RC_DEVICE_ID_VEN_ID__DEV_ID_SHIFT
) |
779 * Set the internal P2P bridge class code.
783 (TRIO_PCIE_RC_REVISION_ID
<<
784 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
785 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD
<<
786 TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
787 (mac
<< TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
789 class_code_revision
=
790 __gxio_mmio_read32(trio_context
->mmio_base_mac
+
792 class_code_revision
= (class_code_revision
& 0xff ) |
793 (PCI_CLASS_BRIDGE_PCI
<< 16);
795 __gxio_mmio_write32(trio_context
->mmio_base_mac
+
796 reg_offset
, class_code_revision
);
798 #ifdef USE_SHARED_PCIE_CONFIG_REGION
801 * Map in the MMIO space for the PIO region.
803 offset
= HV_TRIO_PIO_OFFSET(trio_context
->pio_cfg_index
) |
804 (((unsigned long long)mac
) <<
805 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT
);
810 * Alloc a PIO region for PCI config access per MAC.
812 ret
= gxio_trio_alloc_pio_regions(trio_context
, 1, 0, 0);
814 pr_err("PCI: PCI CFG PIO alloc failure for mac %d "
815 "on TRIO %d, give up\n", mac
, trio_index
);
820 trio_context
->pio_cfg_index
[mac
] = ret
;
823 * For PIO CFG, the bus_address_hi parameter is 0.
825 ret
= gxio_trio_init_pio_region_aux(trio_context
,
826 trio_context
->pio_cfg_index
[mac
],
827 mac
, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE
);
829 pr_err("PCI: PCI CFG PIO init failure for mac %d "
830 "on TRIO %d, give up\n", mac
, trio_index
);
835 offset
= HV_TRIO_PIO_OFFSET(trio_context
->pio_cfg_index
[mac
]) |
836 (((unsigned long long)mac
) <<
837 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT
);
841 trio_context
->mmio_base_pio_cfg
[mac
] =
842 iorpc_ioremap(trio_context
->fd
, offset
,
843 (1 << TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT
));
844 if (trio_context
->mmio_base_pio_cfg
[mac
] == NULL
) {
845 pr_err("PCI: PIO map failure for mac %d on TRIO %d\n",
852 * Initialize the PCIe interrupts.
854 if (tile_init_irqs(controller
)) {
855 pr_err("PCI: IRQs init failure for mac %d on TRIO %d\n",
862 * The PCI memory resource is located above the PA space.
863 * The memory range for the PCI root bus should not overlap
864 * with the physical RAM
866 pci_add_resource_offset(&resources
, &controller
->mem_space
,
867 controller
->mem_offset
);
869 controller
->first_busno
= next_busno
;
870 bus
= pci_scan_root_bus(NULL
, next_busno
, controller
->ops
,
871 controller
, &resources
);
872 controller
->root_bus
= bus
;
873 next_busno
= bus
->busn_res
.end
+ 1;
877 /* Do machine dependent PCI interrupt routing */
878 pci_fixup_irqs(pci_common_swizzle
, tile_map_irq
);
881 * This comes from the generic Linux PCI driver.
883 * It allocates all of the resources (I/O memory, etc)
884 * associated with the devices read in above.
887 pci_assign_unassigned_resources();
889 /* Record the I/O resources in the PCI controller structure. */
890 for (i
= 0; i
< num_rc_controllers
; i
++) {
891 struct pci_controller
*controller
= &pci_controllers
[i
];
892 gxio_trio_context_t
*trio_context
= controller
->trio
;
893 struct pci_bus
*root_bus
= pci_controllers
[i
].root_bus
;
894 struct pci_bus
*next_bus
;
895 uint32_t bus_address_hi
;
901 * Skip controllers that are not properly initialized or
904 if (root_bus
== NULL
)
907 /* Configure the max_payload_size values for this domain. */
908 fixup_read_and_payload_sizes(controller
);
910 list_for_each_entry(dev
, &root_bus
->devices
, bus_list
) {
911 /* Find the PCI host controller, ie. the 1st bridge. */
912 if ((dev
->class >> 8) == PCI_CLASS_BRIDGE_PCI
&&
913 (PCI_SLOT(dev
->devfn
) == 0)) {
914 next_bus
= dev
->subordinate
;
915 pci_controllers
[i
].mem_resources
[0] =
916 *next_bus
->resource
[0];
917 pci_controllers
[i
].mem_resources
[1] =
918 *next_bus
->resource
[1];
919 pci_controllers
[i
].mem_resources
[2] =
920 *next_bus
->resource
[2];
926 if (pci_controllers
[i
].mem_resources
[1].flags
& IORESOURCE_MEM
)
928 pci_controllers
[i
].mem_resources
[1].start
>> 32;
929 else if (pci_controllers
[i
].mem_resources
[2].flags
& IORESOURCE_PREFETCH
)
931 pci_controllers
[i
].mem_resources
[2].start
>> 32;
933 /* This is unlikely. */
934 pr_err("PCI: no memory resources on TRIO %d mac %d\n",
935 controller
->trio_index
, controller
->mac
);
940 * Alloc a PIO region for PCI memory access for each RC port.
942 ret
= gxio_trio_alloc_pio_regions(trio_context
, 1, 0, 0);
944 pr_err("PCI: MEM PIO alloc failure on TRIO %d mac %d, "
945 "give up\n", controller
->trio_index
,
951 controller
->pio_mem_index
= ret
;
954 * For PIO MEM, the bus_address_hi parameter is hard-coded 0
955 * because we always assign 32-bit PCI bus BAR ranges.
957 ret
= gxio_trio_init_pio_region_aux(trio_context
,
958 controller
->pio_mem_index
,
963 pr_err("PCI: MEM PIO init failure on TRIO %d mac %d, "
964 "give up\n", controller
->trio_index
,
971 * Configure a Mem-Map region for each memory controller so
972 * that Linux can map all of its PA space to the PCI bus.
973 * Use the IOMMU to handle hash-for-home memory.
975 for_each_online_node(j
) {
976 unsigned long start_pfn
= node_start_pfn
[j
];
977 unsigned long end_pfn
= node_end_pfn
[j
];
978 unsigned long nr_pages
= end_pfn
- start_pfn
;
980 ret
= gxio_trio_alloc_memory_maps(trio_context
, 1, 0,
983 pr_err("PCI: Mem-Map alloc failure on TRIO %d "
984 "mac %d for MC %d, give up\n",
985 controller
->trio_index
,
988 goto alloc_mem_map_failed
;
991 controller
->mem_maps
[j
] = ret
;
994 * Initialize the Mem-Map and the I/O MMU so that all
995 * the physical memory can be accessed by the endpoint
996 * devices. The base bus address is set to the base CPA
997 * of this memory controller plus an offset (see pci.h).
998 * The region's base VA is set to the base CPA. The
999 * I/O MMU table essentially translates the CPA to
1000 * the real PA. Implicitly, for node 0, we create
1001 * a separate Mem-Map region that serves as the inbound
1002 * window for legacy 32-bit devices. This is a direct
1003 * map of the low 4GB CPA space.
1005 ret
= gxio_trio_init_memory_map_mmu_aux(trio_context
,
1006 controller
->mem_maps
[j
],
1007 start_pfn
<< PAGE_SHIFT
,
1008 nr_pages
<< PAGE_SHIFT
,
1011 (start_pfn
<< PAGE_SHIFT
) +
1012 TILE_PCI_MEM_MAP_BASE_OFFSET
,
1014 GXIO_TRIO_ORDER_MODE_UNORDERED
);
1016 pr_err("PCI: Mem-Map init failure on TRIO %d "
1017 "mac %d for MC %d, give up\n",
1018 controller
->trio_index
,
1019 controller
->mac
, j
);
1021 goto alloc_mem_map_failed
;
1025 alloc_mem_map_failed
:
1033 subsys_initcall(pcibios_init
);
1035 /* Note: to be deleted after Linux 3.6 merge. */
1036 void pcibios_fixup_bus(struct pci_bus
*bus
)
1041 * This can be called from the generic PCI layer, but doesn't need to
1044 char *pcibios_setup(char *str
)
1046 if (!strcmp(str
, "off")) {
1054 * Enable memory address decoding, as appropriate, for the
1055 * device described by the 'dev' struct. The I/O decoding
1056 * is disabled, though the TILE-Gx supports I/O addressing.
1058 * This is called from the generic PCI layer, and can be called
1059 * for bridges or endpoints.
1061 int pcibios_enable_device(struct pci_dev
*dev
, int mask
)
1063 return pci_enable_resources(dev
, mask
);
1066 /* Called for each device after PCI setup is done. */
1067 static void pcibios_fixup_final(struct pci_dev
*pdev
)
1069 set_dma_ops(&pdev
->dev
, gx_pci_dma_map_ops
);
1070 set_dma_offset(&pdev
->dev
, TILE_PCI_MEM_MAP_BASE_OFFSET
);
1071 pdev
->dev
.archdata
.max_direct_dma_addr
=
1072 TILE_PCI_MAX_DIRECT_DMA_ADDRESS
;
1074 DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID
, PCI_ANY_ID
, pcibios_fixup_final
);
1076 /* Map a PCI MMIO bus address into VA space. */
1077 void __iomem
*ioremap(resource_size_t phys_addr
, unsigned long size
)
1079 struct pci_controller
*controller
= NULL
;
1080 resource_size_t bar_start
;
1081 resource_size_t bar_end
;
1082 resource_size_t offset
;
1083 resource_size_t start
;
1084 resource_size_t end
;
1089 end
= phys_addr
+ size
- 1;
1092 * In the following, each PCI controller's mem_resources[1]
1093 * represents its (non-prefetchable) PCI memory resource and
1094 * mem_resources[2] refers to its prefetchable PCI memory resource.
1095 * By searching phys_addr in each controller's mem_resources[], we can
1096 * determine the controller that should accept the PCI memory access.
1099 for (i
= 0; i
< num_rc_controllers
; i
++) {
1101 * Skip controllers that are not properly initialized or
1104 if (pci_controllers
[i
].root_bus
== NULL
)
1107 for (j
= 1; j
< 3; j
++) {
1109 pci_controllers
[i
].mem_resources
[j
].start
;
1111 pci_controllers
[i
].mem_resources
[j
].end
;
1113 if ((start
>= bar_start
) && (end
<= bar_end
)) {
1115 controller
= &pci_controllers
[i
];
1122 if (controller
== NULL
)
1126 trio_fd
= controller
->trio
->fd
;
1128 /* Convert the resource start to the bus address offset. */
1129 start
= phys_addr
- controller
->mem_offset
;
1131 offset
= HV_TRIO_PIO_OFFSET(controller
->pio_mem_index
) + start
;
1134 * We need to keep the PCI bus address's in-page offset in the VA.
1136 return iorpc_ioremap(trio_fd
, offset
, size
) +
1137 (phys_addr
& (PAGE_SIZE
- 1));
1139 EXPORT_SYMBOL(ioremap
);
1141 void pci_iounmap(struct pci_dev
*dev
, void __iomem
*addr
)
1145 EXPORT_SYMBOL(pci_iounmap
);
1147 /****************************************************************
1149 * Tile PCI config space read/write routines
1151 ****************************************************************/
1154 * These are the normal read and write ops
1155 * These are expanded with macros from pci_bus_read_config_byte() etc.
1157 * devfn is the combined PCI device & function.
1159 * offset is in bytes, from the start of config space for the
1160 * specified bus & device.
1163 static int tile_cfg_read(struct pci_bus
*bus
, unsigned int devfn
, int offset
,
1166 struct pci_controller
*controller
= bus
->sysdata
;
1167 gxio_trio_context_t
*trio_context
= controller
->trio
;
1168 int busnum
= bus
->number
& 0xff;
1169 int device
= PCI_SLOT(devfn
);
1170 int function
= PCI_FUNC(devfn
);
1171 int config_type
= 1;
1172 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr
;
1176 * Map all accesses to the local device on root bus into the
1177 * MMIO space of the MAC. Accesses to the downstream devices
1178 * go to the PIO space.
1180 if (pci_is_root_bus(bus
)) {
1183 * This is the internal downstream P2P bridge,
1186 unsigned int reg_offset
;
1188 reg_offset
= ((offset
& 0xFFF) <<
1189 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
1190 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED
1191 << TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
1193 TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
1195 mmio_addr
= trio_context
->mmio_base_mac
+ reg_offset
;
1201 * We fake an empty device for (device > 0),
1202 * since there is only one device on bus 0.
1204 goto invalid_device
;
1209 * Accesses to the directly attached device have to be
1210 * sent as type-0 configs.
1213 if (busnum
== (controller
->first_busno
+ 1)) {
1215 * There is only one device off of our built-in P2P bridge.
1218 goto invalid_device
;
1224 cfg_addr
.reg_addr
= (offset
& 0xFFF);
1225 cfg_addr
.fn
= function
;
1226 cfg_addr
.dev
= device
;
1227 cfg_addr
.bus
= busnum
;
1228 cfg_addr
.type
= config_type
;
1231 * Note that we don't set the mac field in cfg_addr because the
1232 * mapping is per port.
1235 mmio_addr
= trio_context
->mmio_base_pio_cfg
[controller
->mac
] +
1242 *val
= __gxio_mmio_read32(mmio_addr
);
1246 *val
= __gxio_mmio_read16(mmio_addr
);
1250 *val
= __gxio_mmio_read8(mmio_addr
);
1254 return PCIBIOS_FUNC_NOT_SUPPORTED
;
1257 TRACE_CFG_RD(size
, *val
, busnum
, device
, function
, offset
);
1277 return PCIBIOS_FUNC_NOT_SUPPORTED
;
1285 * See tile_cfg_read() for relevent comments.
1286 * Note that "val" is the value to write, not a pointer to that value.
1288 static int tile_cfg_write(struct pci_bus
*bus
, unsigned int devfn
, int offset
,
1291 struct pci_controller
*controller
= bus
->sysdata
;
1292 gxio_trio_context_t
*trio_context
= controller
->trio
;
1293 int busnum
= bus
->number
& 0xff;
1294 int device
= PCI_SLOT(devfn
);
1295 int function
= PCI_FUNC(devfn
);
1296 int config_type
= 1;
1297 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr
;
1299 u32 val_32
= (u32
)val
;
1300 u16 val_16
= (u16
)val
;
1304 * Map all accesses to the local device on root bus into the
1305 * MMIO space of the MAC. Accesses to the downstream devices
1306 * go to the PIO space.
1308 if (pci_is_root_bus(bus
)) {
1311 * This is the internal downstream P2P bridge,
1314 unsigned int reg_offset
;
1316 reg_offset
= ((offset
& 0xFFF) <<
1317 TRIO_CFG_REGION_ADDR__REG_SHIFT
) |
1318 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED
1319 << TRIO_CFG_REGION_ADDR__INTFC_SHIFT
) |
1321 TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT
);
1323 mmio_addr
= trio_context
->mmio_base_mac
+ reg_offset
;
1329 * We fake an empty device for (device > 0),
1330 * since there is only one device on bus 0.
1332 goto invalid_device
;
1337 * Accesses to the directly attached device have to be
1338 * sent as type-0 configs.
1341 if (busnum
== (controller
->first_busno
+ 1)) {
1343 * There is only one device off of our built-in P2P bridge.
1346 goto invalid_device
;
1352 cfg_addr
.reg_addr
= (offset
& 0xFFF);
1353 cfg_addr
.fn
= function
;
1354 cfg_addr
.dev
= device
;
1355 cfg_addr
.bus
= busnum
;
1356 cfg_addr
.type
= config_type
;
1359 * Note that we don't set the mac field in cfg_addr because the
1360 * mapping is per port.
1363 mmio_addr
= trio_context
->mmio_base_pio_cfg
[controller
->mac
] +
1370 __gxio_mmio_write32(mmio_addr
, val_32
);
1371 TRACE_CFG_WR(size
, val_32
, busnum
, device
, function
, offset
);
1375 __gxio_mmio_write16(mmio_addr
, val_16
);
1376 TRACE_CFG_WR(size
, val_16
, busnum
, device
, function
, offset
);
1380 __gxio_mmio_write8(mmio_addr
, val_8
);
1381 TRACE_CFG_WR(size
, val_8
, busnum
, device
, function
, offset
);
1385 return PCIBIOS_FUNC_NOT_SUPPORTED
;
1394 static struct pci_ops tile_cfg_ops
= {
1395 .read
= tile_cfg_read
,
1396 .write
= tile_cfg_write
,
1401 * MSI support starts here.
1404 tilegx_msi_startup(struct irq_data
*d
)
1413 tilegx_msi_ack(struct irq_data
*d
)
1415 __insn_mtspr(SPR_IPI_EVENT_RESET_K
, 1UL << d
->irq
);
1419 tilegx_msi_mask(struct irq_data
*d
)
1422 __insn_mtspr(SPR_IPI_MASK_SET_K
, 1UL << d
->irq
);
1426 tilegx_msi_unmask(struct irq_data
*d
)
1428 __insn_mtspr(SPR_IPI_MASK_RESET_K
, 1UL << d
->irq
);
1432 static struct irq_chip tilegx_msi_chip
= {
1433 .name
= "tilegx_msi",
1434 .irq_startup
= tilegx_msi_startup
,
1435 .irq_ack
= tilegx_msi_ack
,
1436 .irq_mask
= tilegx_msi_mask
,
1437 .irq_unmask
= tilegx_msi_unmask
,
1439 /* TBD: support set_affinity. */
1442 int arch_setup_msi_irq(struct pci_dev
*pdev
, struct msi_desc
*desc
)
1444 struct pci_controller
*controller
;
1445 gxio_trio_context_t
*trio_context
;
1448 uint64_t mem_map_base
;
1449 uint64_t mem_map_limit
;
1461 * Since we use a 64-bit Mem-Map to accept the MSI write, we fail
1462 * devices that are not capable of generating a 64-bit message address.
1463 * These devices will fall back to using the legacy interrupts.
1464 * Most PCIe endpoint devices do support 64-bit message addressing.
1466 if (desc
->msi_attrib
.is_64
== 0) {
1467 dev_printk(KERN_INFO
, &pdev
->dev
,
1468 "64-bit MSI message address not supported, "
1469 "falling back to legacy interrupts.\n");
1475 default_irq
= desc
->msi_attrib
.default_irq
;
1476 controller
= irq_get_handler_data(default_irq
);
1478 BUG_ON(!controller
);
1480 trio_context
= controller
->trio
;
1483 * Allocate a scatter-queue that will accept the MSI write and
1484 * trigger the TILE-side interrupts. We use the scatter-queue regions
1485 * before the mem map regions, because the latter are needed by more
1488 mem_map
= gxio_trio_alloc_scatter_queues(trio_context
, 1, 0, 0);
1490 TRIO_MAP_SQ_DOORBELL_FMT_t doorbell_template
= {{
1495 mem_map
+= TRIO_NUM_MAP_MEM_REGIONS
;
1496 mem_map_base
= MEM_MAP_INTR_REGIONS_BASE
+
1497 mem_map
* MEM_MAP_INTR_REGION_SIZE
;
1498 mem_map_limit
= mem_map_base
+ MEM_MAP_INTR_REGION_SIZE
- 1;
1500 msi_addr
= mem_map_base
+ MEM_MAP_INTR_REGION_SIZE
- 8;
1501 msg
.data
= (unsigned int)doorbell_template
.word
;
1503 /* SQ regions are out, allocate from map mem regions. */
1504 mem_map
= gxio_trio_alloc_memory_maps(trio_context
, 1, 0, 0);
1506 dev_printk(KERN_INFO
, &pdev
->dev
,
1507 "%s Mem-Map alloc failure. "
1508 "Failed to initialize MSI interrupts. "
1509 "Falling back to legacy interrupts.\n",
1510 desc
->msi_attrib
.is_msix
? "MSI-X" : "MSI");
1512 goto msi_mem_map_alloc_failure
;
1515 mem_map_base
= MEM_MAP_INTR_REGIONS_BASE
+
1516 mem_map
* MEM_MAP_INTR_REGION_SIZE
;
1517 mem_map_limit
= mem_map_base
+ MEM_MAP_INTR_REGION_SIZE
- 1;
1519 msi_addr
= mem_map_base
+ TRIO_MAP_MEM_REG_INT3
-
1520 TRIO_MAP_MEM_REG_INT0
;
1525 /* We try to distribute different IRQs to different tiles. */
1526 cpu
= tile_irq_cpu(irq
);
1529 * Now call up to the HV to configure the MSI interrupt and
1530 * set up the IPI binding.
1532 ret
= gxio_trio_config_msi_intr(trio_context
, cpu_x(cpu
), cpu_y(cpu
),
1533 KERNEL_PL
, irq
, controller
->mac
,
1534 mem_map
, mem_map_base
, mem_map_limit
,
1535 trio_context
->asid
);
1537 dev_printk(KERN_INFO
, &pdev
->dev
, "HV MSI config failed.\n");
1539 goto hv_msi_config_failure
;
1542 irq_set_msi_desc(irq
, desc
);
1544 msg
.address_hi
= msi_addr
>> 32;
1545 msg
.address_lo
= msi_addr
& 0xffffffff;
1547 write_msi_msg(irq
, &msg
);
1548 irq_set_chip_and_handler(irq
, &tilegx_msi_chip
, handle_level_irq
);
1549 irq_set_handler_data(irq
, controller
);
1553 hv_msi_config_failure
:
1555 msi_mem_map_alloc_failure
:
1561 void arch_teardown_msi_irq(unsigned int irq
)