diff --git a/sys/arm/allwinner/a10/a10_intc.c b/sys/arm/allwinner/a10/a10_intc.c index 22b29ececac5..410fba4b4728 100644 --- a/sys/arm/allwinner/a10/a10_intc.c +++ b/sys/arm/allwinner/a10/a10_intc.c @@ -1,386 +1,386 @@ /*- * Copyright (c) 2012 Ganbold Tsagaankhuu * Copyright (c) 2016 Emmanuel Vadot * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" /** * Interrupt controller registers * */ #define SW_INT_VECTOR_REG 0x00 #define SW_INT_BASE_ADR_REG 0x04 #define SW_INT_PROTECTION_REG 0x08 #define SW_INT_NMI_CTRL_REG 0x0c #define SW_INT_IRQ_PENDING_REG0 0x10 #define SW_INT_IRQ_PENDING_REG1 0x14 #define SW_INT_IRQ_PENDING_REG2 0x18 #define SW_INT_FIQ_PENDING_REG0 0x20 #define SW_INT_FIQ_PENDING_REG1 0x24 #define SW_INT_FIQ_PENDING_REG2 0x28 #define SW_INT_SELECT_REG0 0x30 #define SW_INT_SELECT_REG1 0x34 #define SW_INT_SELECT_REG2 0x38 #define SW_INT_ENABLE_REG0 0x40 #define SW_INT_ENABLE_REG1 0x44 #define SW_INT_ENABLE_REG2 0x48 #define SW_INT_MASK_REG0 0x50 #define SW_INT_MASK_REG1 0x54 #define SW_INT_MASK_REG2 0x58 #define SW_INT_IRQNO_ENMI 0 #define A10_INTR_MAX_NIRQS 81 #define SW_INT_IRQ_PENDING_REG(_b) (0x10 + ((_b) * 4)) #define SW_INT_FIQ_PENDING_REG(_b) (0x20 + ((_b) * 4)) #define SW_INT_SELECT_REG(_b) (0x30 + ((_b) * 4)) #define SW_INT_ENABLE_REG(_b) (0x40 + ((_b) * 4)) #define SW_INT_MASK_REG(_b) (0x50 + ((_b) * 4)) struct a10_intr_irqsrc { struct intr_irqsrc isrc; u_int irq; }; struct a10_aintc_softc { device_t sc_dev; struct resource * aintc_res; bus_space_tag_t aintc_bst; bus_space_handle_t aintc_bsh; struct mtx mtx; struct a10_intr_irqsrc isrcs[A10_INTR_MAX_NIRQS]; }; #define aintc_read_4(sc, reg) \ bus_space_read_4(sc->aintc_bst, sc->aintc_bsh, reg) #define aintc_write_4(sc, reg, val) \ bus_space_write_4(sc->aintc_bst, sc->aintc_bsh, reg, val) static __inline void a10_intr_eoi(struct a10_aintc_softc *sc, u_int irq) { if (irq != SW_INT_IRQNO_ENMI) return; mtx_lock_spin(&sc->mtx); aintc_write_4(sc, SW_INT_IRQ_PENDING_REG(0), (1 << SW_INT_IRQNO_ENMI)); mtx_unlock_spin(&sc->mtx); } static void a10_intr_unmask(struct a10_aintc_softc *sc, u_int irq) { uint32_t bit, block, value; bit = (irq % 32); block = (irq / 32); mtx_lock_spin(&sc->mtx); value = aintc_read_4(sc, SW_INT_ENABLE_REG(block)); value |= (1 << bit); aintc_write_4(sc, SW_INT_ENABLE_REG(block), value); value = aintc_read_4(sc, SW_INT_MASK_REG(block)); value &= ~(1 << bit); aintc_write_4(sc, SW_INT_MASK_REG(block), value); mtx_unlock_spin(&sc->mtx); } static void a10_intr_mask(struct a10_aintc_softc *sc, u_int irq) { uint32_t bit, block, value; bit = (irq % 32); block = (irq / 32); mtx_lock_spin(&sc->mtx); value = aintc_read_4(sc, SW_INT_ENABLE_REG(block)); value &= ~(1 << bit); aintc_write_4(sc, SW_INT_ENABLE_REG(block), value); value = aintc_read_4(sc, SW_INT_MASK_REG(block)); value |= (1 << bit); aintc_write_4(sc, SW_INT_MASK_REG(block), value); mtx_unlock_spin(&sc->mtx); } static int a10_pending_irq(struct a10_aintc_softc *sc) { uint32_t value; int i, b; for (i = 0; i < 3; i++) { value = aintc_read_4(sc, SW_INT_IRQ_PENDING_REG(i)); if (value == 0) continue; for (b = 0; b < 32; b++) if (value & (1 << b)) { return (i * 32 + b); } } return (-1); } static int a10_intr(void *arg) { struct a10_aintc_softc *sc = arg; u_int irq; irq = a10_pending_irq(sc); if (irq == -1 || irq > A10_INTR_MAX_NIRQS) { device_printf(sc->sc_dev, "Spurious interrupt %d\n", irq); return (FILTER_HANDLED); } while (irq != -1) { if (irq > A10_INTR_MAX_NIRQS) { device_printf(sc->sc_dev, "Spurious interrupt %d\n", irq); return (FILTER_HANDLED); } if (intr_isrc_dispatch(&sc->isrcs[irq].isrc, curthread->td_intr_frame) != 0) { a10_intr_mask(sc, irq); a10_intr_eoi(sc, irq); device_printf(sc->sc_dev, "Stray interrupt %d disabled\n", irq); } arm_irq_memory_barrier(irq); irq = a10_pending_irq(sc); } return (FILTER_HANDLED); } static int a10_intr_pic_attach(struct a10_aintc_softc *sc) { struct intr_pic *pic; int error; uint32_t irq; const char *name; intptr_t xref; name = device_get_nameunit(sc->sc_dev); for (irq = 0; irq < A10_INTR_MAX_NIRQS; irq++) { sc->isrcs[irq].irq = irq; error = intr_isrc_register(&sc->isrcs[irq].isrc, sc->sc_dev, 0, "%s,%u", name, irq); if (error != 0) return (error); } xref = OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)); pic = intr_pic_register(sc->sc_dev, xref); if (pic == NULL) return (ENXIO); - return (intr_pic_claim_root(sc->sc_dev, xref, a10_intr, sc, 0)); + return (intr_pic_claim_root(sc->sc_dev, xref, a10_intr, sc)); } static void a10_intr_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct a10_aintc_softc *sc; u_int irq = ((struct a10_intr_irqsrc *)isrc)->irq; sc = device_get_softc(dev); arm_irq_memory_barrier(irq); a10_intr_unmask(sc, irq); } static void a10_intr_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct a10_aintc_softc *sc; u_int irq = ((struct a10_intr_irqsrc *)isrc)->irq; sc = device_get_softc(dev); a10_intr_mask(sc, irq); } static int a10_intr_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct intr_map_data_fdt *daf; struct a10_aintc_softc *sc; if (data->type != INTR_MAP_DATA_FDT) return (ENOTSUP); daf = (struct intr_map_data_fdt *)data; if (daf->ncells != 1 || daf->cells[0] >= A10_INTR_MAX_NIRQS) return (EINVAL); sc = device_get_softc(dev); *isrcp = &sc->isrcs[daf->cells[0]].isrc; return (0); } static void a10_intr_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct a10_aintc_softc *sc = device_get_softc(dev); u_int irq = ((struct a10_intr_irqsrc *)isrc)->irq; a10_intr_mask(sc, irq); a10_intr_eoi(sc, irq); } static void a10_intr_post_ithread(device_t dev, struct intr_irqsrc *isrc) { a10_intr_enable_intr(dev, isrc); } static void a10_intr_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct a10_aintc_softc *sc = device_get_softc(dev); u_int irq = ((struct a10_intr_irqsrc *)isrc)->irq; a10_intr_eoi(sc, irq); } static int a10_aintc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "allwinner,sun4i-a10-ic")) return (ENXIO); device_set_desc(dev, "A10 AINTC Interrupt Controller"); return (BUS_PROBE_DEFAULT); } static int a10_aintc_attach(device_t dev) { struct a10_aintc_softc *sc = device_get_softc(dev); int rid = 0; int i; sc->sc_dev = dev; sc->aintc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!sc->aintc_res) { device_printf(dev, "could not allocate resource\n"); goto error; } sc->aintc_bst = rman_get_bustag(sc->aintc_res); sc->aintc_bsh = rman_get_bushandle(sc->aintc_res); mtx_init(&sc->mtx, "A10 AINTC lock", "", MTX_SPIN); /* Disable & clear all interrupts */ for (i = 0; i < 3; i++) { aintc_write_4(sc, SW_INT_ENABLE_REG(i), 0); aintc_write_4(sc, SW_INT_MASK_REG(i), 0xffffffff); } /* enable protection mode*/ aintc_write_4(sc, SW_INT_PROTECTION_REG, 0x01); /* config the external interrupt source type*/ aintc_write_4(sc, SW_INT_NMI_CTRL_REG, 0x00); if (a10_intr_pic_attach(sc) != 0) { device_printf(dev, "could not attach PIC\n"); return (ENXIO); } return (0); error: bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->aintc_res); return (ENXIO); } static device_method_t a10_aintc_methods[] = { DEVMETHOD(device_probe, a10_aintc_probe), DEVMETHOD(device_attach, a10_aintc_attach), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, a10_intr_disable_intr), DEVMETHOD(pic_enable_intr, a10_intr_enable_intr), DEVMETHOD(pic_map_intr, a10_intr_map_intr), DEVMETHOD(pic_post_filter, a10_intr_post_filter), DEVMETHOD(pic_post_ithread, a10_intr_post_ithread), DEVMETHOD(pic_pre_ithread, a10_intr_pre_ithread), { 0, 0 } }; static driver_t a10_aintc_driver = { "aintc", a10_aintc_methods, sizeof(struct a10_aintc_softc), }; EARLY_DRIVER_MODULE(aintc, simplebus, a10_aintc_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_FIRST); diff --git a/sys/arm/arm/gic_acpi.c b/sys/arm/arm/gic_acpi.c index 22d8c9f06349..cdd2fc72a224 100644 --- a/sys/arm/arm/gic_acpi.c +++ b/sys/arm/arm/gic_acpi.c @@ -1,353 +1,352 @@ /*- * Copyright (c) 2011,2016 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Developed by Damjan Marion * * Based on OMAP4 GIC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_acpi.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include struct gic_acpi_devinfo { struct resource_list rl; }; static device_identify_t gic_acpi_identify; static device_probe_t gic_acpi_probe; static device_attach_t gic_acpi_attach; static bus_get_resource_list_t gic_acpi_get_resource_list; static bool arm_gic_add_children(device_t); static device_method_t gic_acpi_methods[] = { /* Device interface */ DEVMETHOD(device_identify, gic_acpi_identify), DEVMETHOD(device_probe, gic_acpi_probe), DEVMETHOD(device_attach, gic_acpi_attach), /* Bus interface */ DEVMETHOD(bus_get_resource_list, gic_acpi_get_resource_list), DEVMETHOD_END, }; DEFINE_CLASS_1(gic, gic_acpi_driver, gic_acpi_methods, sizeof(struct arm_gic_softc), arm_gic_driver); EARLY_DRIVER_MODULE(gic, acpi, gic_acpi_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); struct madt_table_data { device_t parent; ACPI_MADT_GENERIC_DISTRIBUTOR *dist; ACPI_MADT_GENERIC_INTERRUPT *intr[GIC_MAXCPU]; }; static void madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct madt_table_data *madt_data; ACPI_MADT_GENERIC_INTERRUPT *intr; madt_data = (struct madt_table_data *)arg; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR: if (madt_data->dist != NULL) { if (bootverbose) device_printf(madt_data->parent, "gic: Already have a distributor table"); } else madt_data->dist = (ACPI_MADT_GENERIC_DISTRIBUTOR *)entry; break; case ACPI_MADT_TYPE_GENERIC_INTERRUPT: intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; if (intr->CpuInterfaceNumber < GIC_MAXCPU) madt_data->intr[intr->CpuInterfaceNumber] = intr; break; } } static void gic_acpi_identify(driver_t *driver, device_t parent) { struct madt_table_data madt_data; ACPI_MADT_GENERIC_INTERRUPT *intr; ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; device_t dev; int i; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return; madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(parent, "gic: Unable to map the MADT\n"); return; } bzero(&madt_data, sizeof(madt_data)); madt_data.parent = parent; madt_data.dist = NULL; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_handler, &madt_data); /* Check the version of the GIC we have */ switch (madt_data.dist->Version) { case ACPI_MADT_GIC_VERSION_NONE: case ACPI_MADT_GIC_VERSION_V1: case ACPI_MADT_GIC_VERSION_V2: break; default: goto out; } intr = NULL; for (i = 0; i < GIC_MAXCPU; i++) { if (madt_data.intr[i] != NULL) { if (intr == NULL) { intr = madt_data.intr[i]; } else if (intr->BaseAddress != madt_data.intr[i]->BaseAddress) { device_printf(parent, "gic: Not all CPU interfaces at the same address, this may fail\n"); } } } if (intr == NULL) { device_printf(parent, "gic: No CPU interfaces found\n"); goto out; } dev = BUS_ADD_CHILD(parent, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE, "gic", -1); if (dev == NULL) { device_printf(parent, "add gic child failed\n"); goto out; } BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 0, madt_data.dist->BaseAddress, 4 * 1024); BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 1, intr->BaseAddress, 4 * 1024); acpi_set_private(dev, (void *)(uintptr_t)madt_data.dist->Version); out: acpi_unmap_table(madt); } static int gic_acpi_probe(device_t dev) { switch((uintptr_t)acpi_get_private(dev)) { case ACPI_MADT_GIC_VERSION_NONE: case ACPI_MADT_GIC_VERSION_V1: case ACPI_MADT_GIC_VERSION_V2: break; default: return (ENXIO); } device_set_desc(dev, "ARM Generic Interrupt Controller"); return (BUS_PROBE_NOWILDCARD); } static int gic_acpi_attach(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); intptr_t xref; int err; sc->gic_bus = GIC_BUS_ACPI; err = arm_gic_attach(dev); if (err != 0) return (err); xref = ACPI_INTR_XREF; /* * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ if (intr_pic_register(dev, xref) == NULL) { device_printf(dev, "could not register PIC\n"); goto cleanup; } /* * Controller is root: */ - if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, - GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { + if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc) != 0) { device_printf(dev, "could not set PIC as a root\n"); intr_pic_deregister(dev, xref); goto cleanup; } /* If we have children probe and attach them */ if (arm_gic_add_children(dev)) { bus_generic_probe(dev); return (bus_generic_attach(dev)); } return (0); cleanup: arm_gic_detach(dev); return(ENXIO); } static struct resource_list * gic_acpi_get_resource_list(device_t bus, device_t child) { struct gic_acpi_devinfo *di; di = device_get_ivars(child); KASSERT(di != NULL, ("gic_acpi_get_resource_list: No devinfo")); return (&di->rl); } static void madt_gicv2m_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct arm_gic_softc *sc; ACPI_MADT_GENERIC_MSI_FRAME *msi; struct gic_acpi_devinfo *dinfo; device_t dev, cdev; if (entry->Type == ACPI_MADT_TYPE_GENERIC_MSI_FRAME) { sc = arg; dev = sc->gic_dev; msi = (ACPI_MADT_GENERIC_MSI_FRAME *)entry; device_printf(dev, "frame: %x %lx %x %u %u\n", msi->MsiFrameId, msi->BaseAddress, msi->Flags, msi->SpiCount, msi->SpiBase); cdev = device_add_child(dev, NULL, -1); if (cdev == NULL) return; dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); resource_list_init(&dinfo->rl); resource_list_add(&dinfo->rl, SYS_RES_MEMORY, 0, msi->BaseAddress, msi->BaseAddress + PAGE_SIZE - 1, PAGE_SIZE); device_set_ivars(cdev, dinfo); } } static bool arm_gic_add_children(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; /* This should return a valid address as it did in gic_acpi_identify */ physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return (false); madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(dev, "gic: Unable to map the MADT\n"); return (false); } acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_gicv2m_handler, sc); acpi_unmap_table(madt); return (true); } static int arm_gicv2m_acpi_probe(device_t dev) { if (gic_get_bus(dev) != GIC_BUS_ACPI) return (EINVAL); if (gic_get_hw_rev(dev) > 2) return (EINVAL); device_set_desc(dev, "ARM Generic Interrupt Controller MSI/MSIX"); return (BUS_PROBE_DEFAULT); } static int arm_gicv2m_acpi_attach(device_t dev) { struct arm_gicv2m_softc *sc; sc = device_get_softc(dev); sc->sc_xref = ACPI_MSI_XREF; return (arm_gicv2m_attach(dev)); } static device_method_t arm_gicv2m_acpi_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gicv2m_acpi_probe), DEVMETHOD(device_attach, arm_gicv2m_acpi_attach), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(gicv2m, arm_gicv2m_acpi_driver, arm_gicv2m_acpi_methods, sizeof(struct arm_gicv2m_softc), arm_gicv2m_driver); EARLY_DRIVER_MODULE(gicv2m_acpi, gic, arm_gicv2m_acpi_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/arm/arm/gic_fdt.c b/sys/arm/arm/gic_fdt.c index f59c9ad79640..c51522b0d58c 100644 --- a/sys/arm/arm/gic_fdt.c +++ b/sys/arm/arm/gic_fdt.c @@ -1,357 +1,356 @@ /*- * Copyright (c) 2011,2016 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Developed by Damjan Marion * * Based on OMAP4 GIC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include struct arm_gic_devinfo { struct ofw_bus_devinfo obdinfo; struct resource_list rl; }; struct arm_gic_fdt_softc { struct arm_gic_softc base; pcell_t addr_cells; pcell_t size_cells; }; static device_probe_t gic_fdt_probe; static device_attach_t gic_fdt_attach; static ofw_bus_get_devinfo_t gic_ofw_get_devinfo; static bus_get_resource_list_t gic_fdt_get_resource_list; static bool arm_gic_add_children(device_t); static struct ofw_compat_data compat_data[] = { {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ {"arm,gic-400", true}, {"arm,cortex-a15-gic", true}, {"arm,cortex-a9-gic", true}, {"arm,cortex-a7-gic", true}, {"arm,arm11mp-gic", true}, {"brcm,brahma-b15-gic", true}, {"qcom,msm-qgic2", true}, {NULL, false} }; static device_method_t gic_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gic_fdt_probe), DEVMETHOD(device_attach, gic_fdt_attach), /* Bus interface */ DEVMETHOD(bus_get_resource_list,gic_fdt_get_resource_list), DEVMETHOD(bus_get_device_path, ofw_bus_gen_get_device_path), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, gic_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), DEVMETHOD_END, }; DEFINE_CLASS_1(gic, gic_fdt_driver, gic_fdt_methods, sizeof(struct arm_gic_fdt_softc), arm_gic_driver); EARLY_DRIVER_MODULE(gic, simplebus, gic_fdt_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic, ofwbus, gic_fdt_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); static int gic_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller"); return (BUS_PROBE_DEFAULT); } static int gic_fdt_attach(device_t dev) { struct arm_gic_fdt_softc *sc = device_get_softc(dev); phandle_t pxref; intptr_t xref; int err; sc->base.gic_bus = GIC_BUS_FDT; err = arm_gic_attach(dev); if (err != 0) return (err); xref = OF_xref_from_node(ofw_bus_get_node(dev)); /* * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ if (intr_pic_register(dev, xref) == NULL) { device_printf(dev, "could not register PIC\n"); goto cleanup; } /* * Controller is root if: * - doesn't have interrupt parent * - his interrupt parent is this controller */ pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); if (pxref == 0 || xref == pxref) { - if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, - GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { + if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc) != 0) { device_printf(dev, "could not set PIC as a root\n"); intr_pic_deregister(dev, xref); goto cleanup; } } else { if (sc->base.gic_res[2] == NULL) { device_printf(dev, "not root PIC must have defined interrupt\n"); intr_pic_deregister(dev, xref); goto cleanup; } if (bus_setup_intr(dev, sc->base.gic_res[2], INTR_TYPE_CLK, arm_gic_intr, NULL, sc, &sc->base.gic_intrhand)) { device_printf(dev, "could not setup irq handler\n"); intr_pic_deregister(dev, xref); goto cleanup; } } OF_device_register_xref(xref, dev); /* If we have children probe and attach them */ if (arm_gic_add_children(dev)) { bus_generic_probe(dev); return (bus_generic_attach(dev)); } return (0); cleanup: arm_gic_detach(dev); return(ENXIO); } static struct resource_list * gic_fdt_get_resource_list(device_t bus, device_t child) { struct arm_gic_devinfo *di; di = device_get_ivars(child); KASSERT(di != NULL, ("gic_fdt_get_resource_list: No devinfo")); return (&di->rl); } static int arm_gic_fill_ranges(phandle_t node, struct arm_gic_fdt_softc *sc) { pcell_t host_cells; cell_t *base_ranges; ssize_t nbase_ranges; int i, j, k; host_cells = 1; OF_getencprop(OF_parent(node), "#address-cells", &host_cells, sizeof(host_cells)); sc->addr_cells = 2; OF_getencprop(node, "#address-cells", &sc->addr_cells, sizeof(sc->addr_cells)); sc->size_cells = 2; OF_getencprop(node, "#size-cells", &sc->size_cells, sizeof(sc->size_cells)); nbase_ranges = OF_getproplen(node, "ranges"); if (nbase_ranges < 0) return (-1); sc->base.nranges = nbase_ranges / sizeof(cell_t) / (sc->addr_cells + host_cells + sc->size_cells); if (sc->base.nranges == 0) return (0); sc->base.ranges = malloc(sc->base.nranges * sizeof(sc->base.ranges[0]), M_DEVBUF, M_WAITOK); base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); OF_getencprop(node, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->base.nranges; i++) { sc->base.ranges[i].bus = 0; for (k = 0; k < sc->addr_cells; k++) { sc->base.ranges[i].bus <<= 32; sc->base.ranges[i].bus |= base_ranges[j++]; } sc->base.ranges[i].host = 0; for (k = 0; k < host_cells; k++) { sc->base.ranges[i].host <<= 32; sc->base.ranges[i].host |= base_ranges[j++]; } sc->base.ranges[i].size = 0; for (k = 0; k < sc->size_cells; k++) { sc->base.ranges[i].size <<= 32; sc->base.ranges[i].size |= base_ranges[j++]; } } free(base_ranges, M_DEVBUF); return (sc->base.nranges); } static bool arm_gic_add_children(device_t dev) { struct arm_gic_fdt_softc *sc; struct arm_gic_devinfo *dinfo; phandle_t child, node; device_t cdev; sc = device_get_softc(dev); node = ofw_bus_get_node(dev); /* If we have no children don't probe for them */ child = OF_child(node); if (child == 0) return (false); if (arm_gic_fill_ranges(node, sc) < 0) { device_printf(dev, "Have a child, but no ranges\n"); return (false); } for (; child != 0; child = OF_peer(child)) { dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&dinfo->obdinfo, child) != 0) { free(dinfo, M_DEVBUF); continue; } resource_list_init(&dinfo->rl); ofw_bus_reg_to_rl(dev, child, sc->addr_cells, sc->size_cells, &dinfo->rl); cdev = device_add_child(dev, NULL, -1); if (cdev == NULL) { device_printf(dev, "<%s>: device_add_child failed\n", dinfo->obdinfo.obd_name); resource_list_free(&dinfo->rl); ofw_bus_gen_destroy_devinfo(&dinfo->obdinfo); free(dinfo, M_DEVBUF); continue; } device_set_ivars(cdev, dinfo); } return (true); } static const struct ofw_bus_devinfo * gic_ofw_get_devinfo(device_t bus __unused, device_t child) { struct arm_gic_devinfo *di; di = device_get_ivars(child); return (&di->obdinfo); } static struct ofw_compat_data gicv2m_compat_data[] = { {"arm,gic-v2m-frame", true}, {NULL, false} }; static int arm_gicv2m_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, gicv2m_compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller MSI/MSIX"); return (BUS_PROBE_DEFAULT); } static int arm_gicv2m_fdt_attach(device_t dev) { struct arm_gicv2m_softc *sc; sc = device_get_softc(dev); sc->sc_xref = OF_xref_from_node(ofw_bus_get_node(dev)); return (arm_gicv2m_attach(dev)); } static device_method_t arm_gicv2m_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gicv2m_fdt_probe), DEVMETHOD(device_attach, arm_gicv2m_fdt_attach), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(gicv2m, arm_gicv2m_fdt_driver, arm_gicv2m_fdt_methods, sizeof(struct arm_gicv2m_softc), arm_gicv2m_driver); EARLY_DRIVER_MODULE(gicv2m, gic, arm_gicv2m_fdt_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/arm/broadcom/bcm2835/bcm2835_intr.c b/sys/arm/broadcom/bcm2835/bcm2835_intr.c index b49c9041810d..5a8457bc4b67 100644 --- a/sys/arm/broadcom/bcm2835/bcm2835_intr.c +++ b/sys/arm/broadcom/bcm2835/bcm2835_intr.c @@ -1,449 +1,449 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 Damjan Marion * All rights reserved. * * Based on OMAP3 INTC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #define INTC_PENDING_BASIC 0x00 #define INTC_PENDING_BANK1 0x04 #define INTC_PENDING_BANK2 0x08 #define INTC_FIQ_CONTROL 0x0C #define INTC_ENABLE_BANK1 0x10 #define INTC_ENABLE_BANK2 0x14 #define INTC_ENABLE_BASIC 0x18 #define INTC_DISABLE_BANK1 0x1C #define INTC_DISABLE_BANK2 0x20 #define INTC_DISABLE_BASIC 0x24 #define INTC_PENDING_BASIC_ARM 0x0000FF #define INTC_PENDING_BASIC_GPU1_PEND 0x000100 #define INTC_PENDING_BASIC_GPU2_PEND 0x000200 #define INTC_PENDING_BASIC_GPU1_7 0x000400 #define INTC_PENDING_BASIC_GPU1_9 0x000800 #define INTC_PENDING_BASIC_GPU1_10 0x001000 #define INTC_PENDING_BASIC_GPU1_18 0x002000 #define INTC_PENDING_BASIC_GPU1_19 0x004000 #define INTC_PENDING_BASIC_GPU2_21 0x008000 #define INTC_PENDING_BASIC_GPU2_22 0x010000 #define INTC_PENDING_BASIC_GPU2_23 0x020000 #define INTC_PENDING_BASIC_GPU2_24 0x040000 #define INTC_PENDING_BASIC_GPU2_25 0x080000 #define INTC_PENDING_BASIC_GPU2_30 0x100000 #define INTC_PENDING_BASIC_MASK 0x1FFFFF #define INTC_PENDING_BASIC_GPU1_MASK (INTC_PENDING_BASIC_GPU1_7 | \ INTC_PENDING_BASIC_GPU1_9 | \ INTC_PENDING_BASIC_GPU1_10 | \ INTC_PENDING_BASIC_GPU1_18 | \ INTC_PENDING_BASIC_GPU1_19) #define INTC_PENDING_BASIC_GPU2_MASK (INTC_PENDING_BASIC_GPU2_21 | \ INTC_PENDING_BASIC_GPU2_22 | \ INTC_PENDING_BASIC_GPU2_23 | \ INTC_PENDING_BASIC_GPU2_24 | \ INTC_PENDING_BASIC_GPU2_25 | \ INTC_PENDING_BASIC_GPU2_30) #define INTC_PENDING_BANK1_MASK (~((1 << 7) | (1 << 9) | (1 << 10) | \ (1 << 18) | (1 << 19))) #define INTC_PENDING_BANK2_MASK (~((1 << 21) | (1 << 22) | (1 << 23) | \ (1 << 24) | (1 << 25) | (1 << 30))) #define BANK1_START 8 #define BANK1_END (BANK1_START + 32 - 1) #define BANK2_START (BANK1_START + 32) #define BANK2_END (BANK2_START + 32 - 1) #define IS_IRQ_BASIC(n) (((n) >= 0) && ((n) < BANK1_START)) #define IS_IRQ_BANK1(n) (((n) >= BANK1_START) && ((n) <= BANK1_END)) #define IS_IRQ_BANK2(n) (((n) >= BANK2_START) && ((n) <= BANK2_END)) #define IRQ_BANK1(n) ((n) - BANK1_START) #define IRQ_BANK2(n) ((n) - BANK2_START) #ifdef DEBUG #define dprintf(fmt, args...) printf(fmt, ##args) #else #define dprintf(fmt, args...) #endif #define BCM_INTC_NIRQS 72 /* 8 + 32 + 32 */ struct bcm_intc_irqsrc { struct intr_irqsrc bii_isrc; u_int bii_irq; uint16_t bii_disable_reg; uint16_t bii_enable_reg; uint32_t bii_mask; }; struct bcm_intc_softc { device_t sc_dev; struct resource * intc_res; bus_space_tag_t intc_bst; bus_space_handle_t intc_bsh; struct resource * intc_irq_res; void * intc_irq_hdl; struct bcm_intc_irqsrc intc_isrcs[BCM_INTC_NIRQS]; }; static struct ofw_compat_data compat_data[] = { {"broadcom,bcm2835-armctrl-ic", 1}, {"brcm,bcm2835-armctrl-ic", 1}, {"brcm,bcm2836-armctrl-ic", 1}, {NULL, 0} }; static struct bcm_intc_softc *bcm_intc_sc = NULL; #define intc_read_4(_sc, reg) \ bus_space_read_4((_sc)->intc_bst, (_sc)->intc_bsh, (reg)) #define intc_write_4(_sc, reg, val) \ bus_space_write_4((_sc)->intc_bst, (_sc)->intc_bsh, (reg), (val)) static inline void bcm_intc_isrc_mask(struct bcm_intc_softc *sc, struct bcm_intc_irqsrc *bii) { intc_write_4(sc, bii->bii_disable_reg, bii->bii_mask); } static inline void bcm_intc_isrc_unmask(struct bcm_intc_softc *sc, struct bcm_intc_irqsrc *bii) { intc_write_4(sc, bii->bii_enable_reg, bii->bii_mask); } static inline int bcm2835_intc_active_intr(struct bcm_intc_softc *sc) { uint32_t pending, pending_gpu; pending = intc_read_4(sc, INTC_PENDING_BASIC) & INTC_PENDING_BASIC_MASK; if (pending == 0) return (-1); if (pending & INTC_PENDING_BASIC_ARM) return (ffs(pending) - 1); if (pending & INTC_PENDING_BASIC_GPU1_MASK) { if (pending & INTC_PENDING_BASIC_GPU1_7) return (BANK1_START + 7); if (pending & INTC_PENDING_BASIC_GPU1_9) return (BANK1_START + 9); if (pending & INTC_PENDING_BASIC_GPU1_10) return (BANK1_START + 10); if (pending & INTC_PENDING_BASIC_GPU1_18) return (BANK1_START + 18); if (pending & INTC_PENDING_BASIC_GPU1_19) return (BANK1_START + 19); } if (pending & INTC_PENDING_BASIC_GPU2_MASK) { if (pending & INTC_PENDING_BASIC_GPU2_21) return (BANK2_START + 21); if (pending & INTC_PENDING_BASIC_GPU2_22) return (BANK2_START + 22); if (pending & INTC_PENDING_BASIC_GPU2_23) return (BANK2_START + 23); if (pending & INTC_PENDING_BASIC_GPU2_24) return (BANK2_START + 24); if (pending & INTC_PENDING_BASIC_GPU2_25) return (BANK2_START + 25); if (pending & INTC_PENDING_BASIC_GPU2_30) return (BANK2_START + 30); } if (pending & INTC_PENDING_BASIC_GPU1_PEND) { pending_gpu = intc_read_4(sc, INTC_PENDING_BANK1); pending_gpu &= INTC_PENDING_BANK1_MASK; if (pending_gpu != 0) return (BANK1_START + ffs(pending_gpu) - 1); } if (pending & INTC_PENDING_BASIC_GPU2_PEND) { pending_gpu = intc_read_4(sc, INTC_PENDING_BANK2); pending_gpu &= INTC_PENDING_BANK2_MASK; if (pending_gpu != 0) return (BANK2_START + ffs(pending_gpu) - 1); } return (-1); /* It shouldn't end here, but it's hardware. */ } static int bcm2835_intc_intr(void *arg) { int irq, num; struct bcm_intc_softc *sc = arg; for (num = 0; ; num++) { irq = bcm2835_intc_active_intr(sc); if (irq == -1) break; if (intr_isrc_dispatch(&sc->intc_isrcs[irq].bii_isrc, curthread->td_intr_frame) != 0) { bcm_intc_isrc_mask(sc, &sc->intc_isrcs[irq]); device_printf(sc->sc_dev, "Stray irq %u disabled\n", irq); } arm_irq_memory_barrier(0); /* XXX */ } if (num == 0 && bootverbose) device_printf(sc->sc_dev, "Spurious interrupt detected\n"); return (FILTER_HANDLED); } static void bcm_intc_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct bcm_intc_irqsrc *bii = (struct bcm_intc_irqsrc *)isrc; arm_irq_memory_barrier(bii->bii_irq); bcm_intc_isrc_unmask(device_get_softc(dev), bii); } static void bcm_intc_disable_intr(device_t dev, struct intr_irqsrc *isrc) { bcm_intc_isrc_mask(device_get_softc(dev), (struct bcm_intc_irqsrc *)isrc); } static int bcm_intc_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { u_int irq; struct intr_map_data_fdt *daf; struct bcm_intc_softc *sc; bool valid; if (data->type != INTR_MAP_DATA_FDT) return (ENOTSUP); daf = (struct intr_map_data_fdt *)data; if (daf->ncells == 1) irq = daf->cells[0]; else if (daf->ncells == 2) { valid = true; switch (daf->cells[0]) { case 0: irq = daf->cells[1]; if (irq >= BANK1_START) valid = false; break; case 1: irq = daf->cells[1] + BANK1_START; if (irq > BANK1_END) valid = false; break; case 2: irq = daf->cells[1] + BANK2_START; if (irq > BANK2_END) valid = false; break; default: valid = false; break; } if (!valid) { device_printf(dev, "invalid IRQ config: bank=%d, irq=%d\n", daf->cells[0], daf->cells[1]); return (EINVAL); } } else return (EINVAL); if (irq >= BCM_INTC_NIRQS) return (EINVAL); sc = device_get_softc(dev); *isrcp = &sc->intc_isrcs[irq].bii_isrc; return (0); } static void bcm_intc_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { bcm_intc_disable_intr(dev, isrc); } static void bcm_intc_post_ithread(device_t dev, struct intr_irqsrc *isrc) { bcm_intc_enable_intr(dev, isrc); } static void bcm_intc_post_filter(device_t dev, struct intr_irqsrc *isrc) { } static int bcm_intc_pic_register(struct bcm_intc_softc *sc, intptr_t xref) { struct bcm_intc_irqsrc *bii; int error; uint32_t irq; const char *name; name = device_get_nameunit(sc->sc_dev); for (irq = 0; irq < BCM_INTC_NIRQS; irq++) { bii = &sc->intc_isrcs[irq]; bii->bii_irq = irq; if (IS_IRQ_BASIC(irq)) { bii->bii_disable_reg = INTC_DISABLE_BASIC; bii->bii_enable_reg = INTC_ENABLE_BASIC; bii->bii_mask = 1 << irq; } else if (IS_IRQ_BANK1(irq)) { bii->bii_disable_reg = INTC_DISABLE_BANK1; bii->bii_enable_reg = INTC_ENABLE_BANK1; bii->bii_mask = 1 << IRQ_BANK1(irq); } else if (IS_IRQ_BANK2(irq)) { bii->bii_disable_reg = INTC_DISABLE_BANK2; bii->bii_enable_reg = INTC_ENABLE_BANK2; bii->bii_mask = 1 << IRQ_BANK2(irq); } else return (ENXIO); error = intr_isrc_register(&bii->bii_isrc, sc->sc_dev, 0, "%s,%u", name, irq); if (error != 0) return (error); } if (intr_pic_register(sc->sc_dev, xref) == NULL) return (ENXIO); return (0); } static int bcm_intc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) return (ENXIO); device_set_desc(dev, "BCM2835 Interrupt Controller"); return (BUS_PROBE_DEFAULT); } static int bcm_intc_attach(device_t dev) { struct bcm_intc_softc *sc = device_get_softc(dev); int rid = 0; intptr_t xref; sc->sc_dev = dev; if (bcm_intc_sc) return (ENXIO); sc->intc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->intc_res == NULL) { device_printf(dev, "could not allocate memory resource\n"); return (ENXIO); } xref = OF_xref_from_node(ofw_bus_get_node(dev)); if (bcm_intc_pic_register(sc, xref) != 0) { bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->intc_res); device_printf(dev, "could not register PIC\n"); return (ENXIO); } rid = 0; sc->intc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->intc_irq_res == NULL) { - if (intr_pic_claim_root(dev, xref, bcm2835_intc_intr, sc, 0) != 0) { + if (intr_pic_claim_root(dev, xref, bcm2835_intc_intr, sc) != 0) { /* XXX clean up */ device_printf(dev, "could not set PIC as a root\n"); return (ENXIO); } } else { if (bus_setup_intr(dev, sc->intc_irq_res, INTR_TYPE_CLK, bcm2835_intc_intr, NULL, sc, &sc->intc_irq_hdl)) { /* XXX clean up */ device_printf(dev, "could not setup irq handler\n"); return (ENXIO); } } sc->intc_bst = rman_get_bustag(sc->intc_res); sc->intc_bsh = rman_get_bushandle(sc->intc_res); bcm_intc_sc = sc; return (0); } static device_method_t bcm_intc_methods[] = { DEVMETHOD(device_probe, bcm_intc_probe), DEVMETHOD(device_attach, bcm_intc_attach), DEVMETHOD(pic_disable_intr, bcm_intc_disable_intr), DEVMETHOD(pic_enable_intr, bcm_intc_enable_intr), DEVMETHOD(pic_map_intr, bcm_intc_map_intr), DEVMETHOD(pic_post_filter, bcm_intc_post_filter), DEVMETHOD(pic_post_ithread, bcm_intc_post_ithread), DEVMETHOD(pic_pre_ithread, bcm_intc_pre_ithread), { 0, 0 } }; static driver_t bcm_intc_driver = { "intc", bcm_intc_methods, sizeof(struct bcm_intc_softc), }; EARLY_DRIVER_MODULE(intc, simplebus, bcm_intc_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/arm/broadcom/bcm2835/bcm2836.c b/sys/arm/broadcom/bcm2835/bcm2836.c index 60734abb40e3..f72bd9075982 100644 --- a/sys/arm/broadcom/bcm2835/bcm2836.c +++ b/sys/arm/broadcom/bcm2835/bcm2836.c @@ -1,741 +1,741 @@ /* * Copyright 2015 Andrew Turner. * Copyright 2016 Svatopluk Kraus * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #endif #include #include #include #ifdef SMP #include #endif #include #include #include "pic_if.h" #define BCM_LINTC_CONTROL_REG 0x00 #define BCM_LINTC_PRESCALER_REG 0x08 #define BCM_LINTC_GPU_ROUTING_REG 0x0c #define BCM_LINTC_PMU_ROUTING_SET_REG 0x10 #define BCM_LINTC_PMU_ROUTING_CLR_REG 0x14 #define BCM_LINTC_TIMER_CFG_REG(n) (0x40 + (n) * 4) #define BCM_LINTC_MBOX_CFG_REG(n) (0x50 + (n) * 4) #define BCM_LINTC_PENDING_REG(n) (0x60 + (n) * 4) #define BCM_LINTC_MBOX0_SET_REG(n) (0x80 + (n) * 16) #define BCM_LINTC_MBOX1_SET_REG(n) (0x84 + (n) * 16) #define BCM_LINTC_MBOX2_SET_REG(n) (0x88 + (n) * 16) #define BCM_LINTC_MBOX3_SET_REG(n) (0x8C + (n) * 16) #define BCM_LINTC_MBOX0_CLR_REG(n) (0xC0 + (n) * 16) #define BCM_LINTC_MBOX1_CLR_REG(n) (0xC4 + (n) * 16) #define BCM_LINTC_MBOX2_CLR_REG(n) (0xC8 + (n) * 16) #define BCM_LINTC_MBOX3_CLR_REG(n) (0xCC + (n) * 16) /* Prescaler Register */ #define BCM_LINTC_PSR_19_2 0x80000000 /* 19.2 MHz */ /* GPU Interrupt Routing Register */ #define BCM_LINTC_GIRR_IRQ_CORE(n) (n) #define BCM_LINTC_GIRR_FIQ_CORE(n) ((n) << 2) /* PMU Interrupt Routing Register */ #define BCM_LINTC_PIRR_IRQ_EN_CORE(n) (1 << (n)) #define BCM_LINTC_PIRR_FIQ_EN_CORE(n) (1 << ((n) + 4)) /* Timer Config Register */ #define BCM_LINTC_TCR_IRQ_EN_TIMER(n) (1 << (n)) #define BCM_LINTC_TCR_FIQ_EN_TIMER(n) (1 << ((n) + 4)) /* MBOX Config Register */ #define BCM_LINTC_MCR_IRQ_EN_MBOX(n) (1 << (n)) #define BCM_LINTC_MCR_FIQ_EN_MBOX(n) (1 << ((n) + 4)) #define BCM_LINTC_CNTPSIRQ_IRQ 0 #define BCM_LINTC_CNTPNSIRQ_IRQ 1 #define BCM_LINTC_CNTHPIRQ_IRQ 2 #define BCM_LINTC_CNTVIRQ_IRQ 3 #define BCM_LINTC_MBOX0_IRQ 4 #define BCM_LINTC_MBOX1_IRQ 5 #define BCM_LINTC_MBOX2_IRQ 6 #define BCM_LINTC_MBOX3_IRQ 7 #define BCM_LINTC_GPU_IRQ 8 #define BCM_LINTC_PMU_IRQ 9 #define BCM_LINTC_AXI_IRQ 10 #define BCM_LINTC_LTIMER_IRQ 11 #define BCM_LINTC_NIRQS 12 #define BCM_LINTC_TIMER0_IRQ BCM_LINTC_CNTPSIRQ_IRQ #define BCM_LINTC_TIMER1_IRQ BCM_LINTC_CNTPNSIRQ_IRQ #define BCM_LINTC_TIMER2_IRQ BCM_LINTC_CNTHPIRQ_IRQ #define BCM_LINTC_TIMER3_IRQ BCM_LINTC_CNTVIRQ_IRQ #define BCM_LINTC_TIMER0_IRQ_MASK (1 << BCM_LINTC_TIMER0_IRQ) #define BCM_LINTC_TIMER1_IRQ_MASK (1 << BCM_LINTC_TIMER1_IRQ) #define BCM_LINTC_TIMER2_IRQ_MASK (1 << BCM_LINTC_TIMER2_IRQ) #define BCM_LINTC_TIMER3_IRQ_MASK (1 << BCM_LINTC_TIMER3_IRQ) #define BCM_LINTC_MBOX0_IRQ_MASK (1 << BCM_LINTC_MBOX0_IRQ) #define BCM_LINTC_GPU_IRQ_MASK (1 << BCM_LINTC_GPU_IRQ) #define BCM_LINTC_PMU_IRQ_MASK (1 << BCM_LINTC_PMU_IRQ) #define BCM_LINTC_UP_PENDING_MASK \ (BCM_LINTC_TIMER0_IRQ_MASK | \ BCM_LINTC_TIMER1_IRQ_MASK | \ BCM_LINTC_TIMER2_IRQ_MASK | \ BCM_LINTC_TIMER3_IRQ_MASK | \ BCM_LINTC_GPU_IRQ_MASK | \ BCM_LINTC_PMU_IRQ_MASK) #define BCM_LINTC_SMP_PENDING_MASK \ (BCM_LINTC_UP_PENDING_MASK | \ BCM_LINTC_MBOX0_IRQ_MASK) #ifdef SMP #define BCM_LINTC_PENDING_MASK BCM_LINTC_SMP_PENDING_MASK #else #define BCM_LINTC_PENDING_MASK BCM_LINTC_UP_PENDING_MASK #endif struct bcm_lintc_irqsrc { struct intr_irqsrc bli_isrc; u_int bli_irq; union { u_int bli_mask; /* for timers */ u_int bli_value; /* for GPU */ }; }; struct bcm_lintc_softc { device_t bls_dev; struct mtx bls_mtx; struct resource * bls_mem; bus_space_tag_t bls_bst; bus_space_handle_t bls_bsh; struct bcm_lintc_irqsrc bls_isrcs[BCM_LINTC_NIRQS]; }; static struct bcm_lintc_softc *bcm_lintc_sc; #ifdef SMP #define BCM_LINTC_NIPIS 32 /* only mailbox 0 is used for IPI */ CTASSERT(INTR_IPI_COUNT <= BCM_LINTC_NIPIS); #endif #define BCM_LINTC_LOCK(sc) mtx_lock_spin(&(sc)->bls_mtx) #define BCM_LINTC_UNLOCK(sc) mtx_unlock_spin(&(sc)->bls_mtx) #define BCM_LINTC_LOCK_INIT(sc) mtx_init(&(sc)->bls_mtx, \ device_get_nameunit((sc)->bls_dev), "bmc_local_intc", MTX_SPIN) #define BCM_LINTC_LOCK_DESTROY(sc) mtx_destroy(&(sc)->bls_mtx) #define bcm_lintc_read_4(sc, reg) \ bus_space_read_4((sc)->bls_bst, (sc)->bls_bsh, (reg)) #define bcm_lintc_write_4(sc, reg, val) \ bus_space_write_4((sc)->bls_bst, (sc)->bls_bsh, (reg), (val)) static inline void bcm_lintc_rwreg_clr(struct bcm_lintc_softc *sc, uint32_t reg, uint32_t mask) { bcm_lintc_write_4(sc, reg, bcm_lintc_read_4(sc, reg) & ~mask); } static inline void bcm_lintc_rwreg_set(struct bcm_lintc_softc *sc, uint32_t reg, uint32_t mask) { bcm_lintc_write_4(sc, reg, bcm_lintc_read_4(sc, reg) | mask); } static void bcm_lintc_timer_mask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { cpuset_t *cpus; uint32_t cpu; cpus = &bli->bli_isrc.isrc_cpu; BCM_LINTC_LOCK(sc); for (cpu = 0; cpu < 4; cpu++) if (CPU_ISSET(cpu, cpus)) bcm_lintc_rwreg_clr(sc, BCM_LINTC_TIMER_CFG_REG(cpu), bli->bli_mask); BCM_LINTC_UNLOCK(sc); } static void bcm_lintc_timer_unmask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { cpuset_t *cpus; uint32_t cpu; cpus = &bli->bli_isrc.isrc_cpu; BCM_LINTC_LOCK(sc); for (cpu = 0; cpu < 4; cpu++) if (CPU_ISSET(cpu, cpus)) bcm_lintc_rwreg_set(sc, BCM_LINTC_TIMER_CFG_REG(cpu), bli->bli_mask); BCM_LINTC_UNLOCK(sc); } static inline void bcm_lintc_gpu_mask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { /* It's accessed just and only by one core. */ bcm_lintc_write_4(sc, BCM_LINTC_GPU_ROUTING_REG, 0); } static inline void bcm_lintc_gpu_unmask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { /* It's accessed just and only by one core. */ bcm_lintc_write_4(sc, BCM_LINTC_GPU_ROUTING_REG, bli->bli_value); } static inline void bcm_lintc_pmu_mask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { cpuset_t *cpus; uint32_t cpu, mask; mask = 0; cpus = &bli->bli_isrc.isrc_cpu; BCM_LINTC_LOCK(sc); for (cpu = 0; cpu < 4; cpu++) if (CPU_ISSET(cpu, cpus)) mask |= BCM_LINTC_PIRR_IRQ_EN_CORE(cpu); /* Write-clear register. */ bcm_lintc_write_4(sc, BCM_LINTC_PMU_ROUTING_CLR_REG, mask); BCM_LINTC_UNLOCK(sc); } static inline void bcm_lintc_pmu_unmask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { cpuset_t *cpus; uint32_t cpu, mask; mask = 0; cpus = &bli->bli_isrc.isrc_cpu; BCM_LINTC_LOCK(sc); for (cpu = 0; cpu < 4; cpu++) if (CPU_ISSET(cpu, cpus)) mask |= BCM_LINTC_PIRR_IRQ_EN_CORE(cpu); /* Write-set register. */ bcm_lintc_write_4(sc, BCM_LINTC_PMU_ROUTING_SET_REG, mask); BCM_LINTC_UNLOCK(sc); } static void bcm_lintc_mask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { switch (bli->bli_irq) { case BCM_LINTC_TIMER0_IRQ: case BCM_LINTC_TIMER1_IRQ: case BCM_LINTC_TIMER2_IRQ: case BCM_LINTC_TIMER3_IRQ: bcm_lintc_timer_mask(sc, bli); return; case BCM_LINTC_MBOX0_IRQ: case BCM_LINTC_MBOX1_IRQ: case BCM_LINTC_MBOX2_IRQ: case BCM_LINTC_MBOX3_IRQ: return; case BCM_LINTC_GPU_IRQ: bcm_lintc_gpu_mask(sc, bli); return; case BCM_LINTC_PMU_IRQ: bcm_lintc_pmu_mask(sc, bli); return; default: panic("%s: not implemented for irq %u", __func__, bli->bli_irq); } } static void bcm_lintc_unmask(struct bcm_lintc_softc *sc, struct bcm_lintc_irqsrc *bli) { switch (bli->bli_irq) { case BCM_LINTC_TIMER0_IRQ: case BCM_LINTC_TIMER1_IRQ: case BCM_LINTC_TIMER2_IRQ: case BCM_LINTC_TIMER3_IRQ: bcm_lintc_timer_unmask(sc, bli); return; case BCM_LINTC_MBOX0_IRQ: case BCM_LINTC_MBOX1_IRQ: case BCM_LINTC_MBOX2_IRQ: case BCM_LINTC_MBOX3_IRQ: return; case BCM_LINTC_GPU_IRQ: bcm_lintc_gpu_unmask(sc, bli); return; case BCM_LINTC_PMU_IRQ: bcm_lintc_pmu_unmask(sc, bli); return; default: panic("%s: not implemented for irq %u", __func__, bli->bli_irq); } } #ifdef SMP static inline void bcm_lintc_ipi_write(struct bcm_lintc_softc *sc, cpuset_t cpus, u_int ipi) { u_int cpu; uint32_t mask; mask = 1 << ipi; for (cpu = 0; cpu < mp_ncpus; cpu++) if (CPU_ISSET(cpu, &cpus)) bcm_lintc_write_4(sc, BCM_LINTC_MBOX0_SET_REG(cpu), mask); } static inline void bcm_lintc_ipi_dispatch(struct bcm_lintc_softc *sc, u_int cpu, struct trapframe *tf) { u_int ipi; uint32_t mask; mask = bcm_lintc_read_4(sc, BCM_LINTC_MBOX0_CLR_REG(cpu)); if (mask == 0) { device_printf(sc->bls_dev, "Spurious ipi detected\n"); return; } for (ipi = 0; mask != 0; mask >>= 1, ipi++) { if ((mask & 0x01) == 0) continue; /* * Clear an IPI before dispatching to not miss anyone * and make sure that it's observed by everybody. */ bcm_lintc_write_4(sc, BCM_LINTC_MBOX0_CLR_REG(cpu), 1 << ipi); #if defined(__aarch64__) dsb(sy); #else dsb(); #endif intr_ipi_dispatch(ipi); } } #endif static inline void bcm_lintc_irq_dispatch(struct bcm_lintc_softc *sc, u_int irq, struct trapframe *tf) { struct bcm_lintc_irqsrc *bli; bli = &sc->bls_isrcs[irq]; if (intr_isrc_dispatch(&bli->bli_isrc, tf) != 0) device_printf(sc->bls_dev, "Stray irq %u detected\n", irq); } static int bcm_lintc_intr(void *arg) { struct bcm_lintc_softc *sc; u_int cpu; uint32_t num, reg; struct trapframe *tf; sc = arg; cpu = PCPU_GET(cpuid); tf = curthread->td_intr_frame; for (num = 0; ; num++) { reg = bcm_lintc_read_4(sc, BCM_LINTC_PENDING_REG(cpu)); if ((reg & BCM_LINTC_PENDING_MASK) == 0) break; #ifdef SMP if (reg & BCM_LINTC_MBOX0_IRQ_MASK) bcm_lintc_ipi_dispatch(sc, cpu, tf); #endif if (reg & BCM_LINTC_TIMER0_IRQ_MASK) bcm_lintc_irq_dispatch(sc, BCM_LINTC_TIMER0_IRQ, tf); if (reg & BCM_LINTC_TIMER1_IRQ_MASK) bcm_lintc_irq_dispatch(sc, BCM_LINTC_TIMER1_IRQ, tf); if (reg & BCM_LINTC_TIMER2_IRQ_MASK) bcm_lintc_irq_dispatch(sc, BCM_LINTC_TIMER2_IRQ, tf); if (reg & BCM_LINTC_TIMER3_IRQ_MASK) bcm_lintc_irq_dispatch(sc, BCM_LINTC_TIMER3_IRQ, tf); if (reg & BCM_LINTC_GPU_IRQ_MASK) bcm_lintc_irq_dispatch(sc, BCM_LINTC_GPU_IRQ, tf); if (reg & BCM_LINTC_PMU_IRQ_MASK) bcm_lintc_irq_dispatch(sc, BCM_LINTC_PMU_IRQ, tf); arm_irq_memory_barrier(0); /* XXX */ } reg &= ~BCM_LINTC_PENDING_MASK; if (reg != 0) device_printf(sc->bls_dev, "Unknown interrupt(s) %x\n", reg); else if (num == 0 && bootverbose) device_printf(sc->bls_dev, "Spurious interrupt detected\n"); return (FILTER_HANDLED); } static void bcm_lintc_disable_intr(device_t dev, struct intr_irqsrc *isrc) { bcm_lintc_mask(device_get_softc(dev), (struct bcm_lintc_irqsrc *)isrc); } static void bcm_lintc_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct bcm_lintc_irqsrc *bli = (struct bcm_lintc_irqsrc *)isrc; arm_irq_memory_barrier(bli->bli_irq); bcm_lintc_unmask(device_get_softc(dev), bli); } static int bcm_lintc_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct intr_map_data_fdt *daf; struct bcm_lintc_softc *sc; if (data->type != INTR_MAP_DATA_FDT) return (ENOTSUP); daf = (struct intr_map_data_fdt *)data; if (daf->ncells > 2 || daf->cells[0] >= BCM_LINTC_NIRQS) return (EINVAL); /* TODO: handle IRQ type here */ sc = device_get_softc(dev); *isrcp = &sc->bls_isrcs[daf->cells[0]].bli_isrc; return (0); } static void bcm_lintc_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct bcm_lintc_irqsrc *bli = (struct bcm_lintc_irqsrc *)isrc; if (bli->bli_irq == BCM_LINTC_GPU_IRQ) bcm_lintc_gpu_mask(device_get_softc(dev), bli); else { /* * Handler for PPI interrupt does not make sense much unless * there is one bound ithread for each core for it. Thus the * interrupt can be masked on current core only while ithread * bounded to this core ensures unmasking on the same core. */ panic ("%s: handlers are not supported", __func__); } } static void bcm_lintc_post_ithread(device_t dev, struct intr_irqsrc *isrc) { struct bcm_lintc_irqsrc *bli = (struct bcm_lintc_irqsrc *)isrc; if (bli->bli_irq == BCM_LINTC_GPU_IRQ) bcm_lintc_gpu_unmask(device_get_softc(dev), bli); else { /* See comment in bcm_lintc_pre_ithread(). */ panic ("%s: handlers are not supported", __func__); } } static void bcm_lintc_post_filter(device_t dev, struct intr_irqsrc *isrc) { } static int bcm_lintc_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct bcm_lintc_softc *sc; if (isrc->isrc_handlers == 0 && isrc->isrc_flags & INTR_ISRCF_PPI) { sc = device_get_softc(dev); BCM_LINTC_LOCK(sc); CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); BCM_LINTC_UNLOCK(sc); } return (0); } #ifdef SMP static void bcm_lintc_init_rwreg_on_ap(struct bcm_lintc_softc *sc, u_int cpu, u_int irq, uint32_t reg, uint32_t mask) { if (intr_isrc_init_on_cpu(&sc->bls_isrcs[irq].bli_isrc, cpu)) bcm_lintc_rwreg_set(sc, reg, mask); } static void bcm_lintc_init_pmu_on_ap(struct bcm_lintc_softc *sc, u_int cpu) { struct intr_irqsrc *isrc = &sc->bls_isrcs[BCM_LINTC_PMU_IRQ].bli_isrc; if (intr_isrc_init_on_cpu(isrc, cpu)) { /* Write-set register. */ bcm_lintc_write_4(sc, BCM_LINTC_PMU_ROUTING_SET_REG, BCM_LINTC_PIRR_IRQ_EN_CORE(cpu)); } } static void bcm_lintc_init_secondary(device_t dev) { u_int cpu; struct bcm_lintc_softc *sc; cpu = PCPU_GET(cpuid); sc = device_get_softc(dev); BCM_LINTC_LOCK(sc); bcm_lintc_init_rwreg_on_ap(sc, cpu, BCM_LINTC_TIMER0_IRQ, BCM_LINTC_TIMER_CFG_REG(cpu), BCM_LINTC_TCR_IRQ_EN_TIMER(0)); bcm_lintc_init_rwreg_on_ap(sc, cpu, BCM_LINTC_TIMER1_IRQ, BCM_LINTC_TIMER_CFG_REG(cpu), BCM_LINTC_TCR_IRQ_EN_TIMER(1)); bcm_lintc_init_rwreg_on_ap(sc, cpu, BCM_LINTC_TIMER2_IRQ, BCM_LINTC_TIMER_CFG_REG(cpu), BCM_LINTC_TCR_IRQ_EN_TIMER(2)); bcm_lintc_init_rwreg_on_ap(sc, cpu, BCM_LINTC_TIMER3_IRQ, BCM_LINTC_TIMER_CFG_REG(cpu), BCM_LINTC_TCR_IRQ_EN_TIMER(3)); bcm_lintc_init_pmu_on_ap(sc, cpu); BCM_LINTC_UNLOCK(sc); } static void bcm_lintc_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, u_int ipi) { struct bcm_lintc_softc *sc = device_get_softc(dev); KASSERT(isrc == &sc->bls_isrcs[BCM_LINTC_MBOX0_IRQ].bli_isrc, ("%s: bad ISRC %p argument", __func__, isrc)); bcm_lintc_ipi_write(sc, cpus, ipi); } static int bcm_lintc_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) { struct bcm_lintc_softc *sc = device_get_softc(dev); KASSERT(ipi < BCM_LINTC_NIPIS, ("%s: too high ipi %u", __func__, ipi)); *isrcp = &sc->bls_isrcs[BCM_LINTC_MBOX0_IRQ].bli_isrc; return (0); } #endif static int bcm_lintc_pic_attach(struct bcm_lintc_softc *sc) { struct bcm_lintc_irqsrc *bisrcs; struct intr_pic *pic; int error; u_int flags; uint32_t irq; const char *name; intptr_t xref; bisrcs = sc->bls_isrcs; name = device_get_nameunit(sc->bls_dev); for (irq = 0; irq < BCM_LINTC_NIRQS; irq++) { bisrcs[irq].bli_irq = irq; switch (irq) { case BCM_LINTC_TIMER0_IRQ: bisrcs[irq].bli_mask = BCM_LINTC_TCR_IRQ_EN_TIMER(0); flags = INTR_ISRCF_PPI; break; case BCM_LINTC_TIMER1_IRQ: bisrcs[irq].bli_mask = BCM_LINTC_TCR_IRQ_EN_TIMER(1); flags = INTR_ISRCF_PPI; break; case BCM_LINTC_TIMER2_IRQ: bisrcs[irq].bli_mask = BCM_LINTC_TCR_IRQ_EN_TIMER(2); flags = INTR_ISRCF_PPI; break; case BCM_LINTC_TIMER3_IRQ: bisrcs[irq].bli_mask = BCM_LINTC_TCR_IRQ_EN_TIMER(3); flags = INTR_ISRCF_PPI; break; case BCM_LINTC_MBOX0_IRQ: case BCM_LINTC_MBOX1_IRQ: case BCM_LINTC_MBOX2_IRQ: case BCM_LINTC_MBOX3_IRQ: bisrcs[irq].bli_value = 0; /* not used */ flags = INTR_ISRCF_IPI; break; case BCM_LINTC_GPU_IRQ: bisrcs[irq].bli_value = BCM_LINTC_GIRR_IRQ_CORE(0); flags = 0; break; case BCM_LINTC_PMU_IRQ: bisrcs[irq].bli_value = 0; /* not used */ flags = INTR_ISRCF_PPI; break; default: bisrcs[irq].bli_value = 0; /* not used */ flags = 0; break; } error = intr_isrc_register(&bisrcs[irq].bli_isrc, sc->bls_dev, flags, "%s,%u", name, irq); if (error != 0) return (error); } xref = OF_xref_from_node(ofw_bus_get_node(sc->bls_dev)); pic = intr_pic_register(sc->bls_dev, xref); if (pic == NULL) return (ENXIO); - return (intr_pic_claim_root(sc->bls_dev, xref, bcm_lintc_intr, sc, 0)); + return (intr_pic_claim_root(sc->bls_dev, xref, bcm_lintc_intr, sc)); } static int bcm_lintc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "brcm,bcm2836-l1-intc")) return (ENXIO); if (!ofw_bus_has_prop(dev, "interrupt-controller")) return (ENXIO); device_set_desc(dev, "BCM2836 Interrupt Controller"); return (BUS_PROBE_DEFAULT); } static int bcm_lintc_attach(device_t dev) { struct bcm_lintc_softc *sc; int cpu, rid; sc = device_get_softc(dev); sc->bls_dev = dev; if (bcm_lintc_sc != NULL) return (ENXIO); rid = 0; sc->bls_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->bls_mem == NULL) { device_printf(dev, "could not allocate memory resource\n"); return (ENXIO); } sc->bls_bst = rman_get_bustag(sc->bls_mem); sc->bls_bsh = rman_get_bushandle(sc->bls_mem); bcm_lintc_write_4(sc, BCM_LINTC_CONTROL_REG, 0); bcm_lintc_write_4(sc, BCM_LINTC_PRESCALER_REG, BCM_LINTC_PSR_19_2); /* Disable all timers on all cores. */ for (cpu = 0; cpu < 4; cpu++) bcm_lintc_write_4(sc, BCM_LINTC_TIMER_CFG_REG(cpu), 0); #ifdef SMP /* Enable mailbox 0 on all cores used for IPI. */ for (cpu = 0; cpu < 4; cpu++) bcm_lintc_write_4(sc, BCM_LINTC_MBOX_CFG_REG(cpu), BCM_LINTC_MCR_IRQ_EN_MBOX(0)); #endif if (bcm_lintc_pic_attach(sc) != 0) { device_printf(dev, "could not attach PIC\n"); return (ENXIO); } BCM_LINTC_LOCK_INIT(sc); bcm_lintc_sc = sc; return (0); } static device_method_t bcm_lintc_methods[] = { DEVMETHOD(device_probe, bcm_lintc_probe), DEVMETHOD(device_attach, bcm_lintc_attach), DEVMETHOD(pic_disable_intr, bcm_lintc_disable_intr), DEVMETHOD(pic_enable_intr, bcm_lintc_enable_intr), DEVMETHOD(pic_map_intr, bcm_lintc_map_intr), DEVMETHOD(pic_post_filter, bcm_lintc_post_filter), DEVMETHOD(pic_post_ithread, bcm_lintc_post_ithread), DEVMETHOD(pic_pre_ithread, bcm_lintc_pre_ithread), DEVMETHOD(pic_setup_intr, bcm_lintc_setup_intr), #ifdef SMP DEVMETHOD(pic_init_secondary, bcm_lintc_init_secondary), DEVMETHOD(pic_ipi_send, bcm_lintc_ipi_send), DEVMETHOD(pic_ipi_setup, bcm_lintc_ipi_setup), #endif DEVMETHOD_END }; static driver_t bcm_lintc_driver = { "lintc", bcm_lintc_methods, sizeof(struct bcm_lintc_softc), }; EARLY_DRIVER_MODULE(lintc, simplebus, bcm_lintc_driver, 0, 0, BUS_PASS_INTERRUPT); diff --git a/sys/arm/ti/aintc.c b/sys/arm/ti/aintc.c index ce986b2d38bc..776d231c3c48 100644 --- a/sys/arm/ti/aintc.c +++ b/sys/arm/ti/aintc.c @@ -1,305 +1,305 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 Damjan Marion * All rights reserved. * * Based on OMAP3 INTC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #define INTC_REVISION 0x00 #define INTC_SYSCONFIG 0x10 #define INTC_SYSSTATUS 0x14 #define INTC_SIR_IRQ 0x40 #define INTC_CONTROL 0x48 #define INTC_THRESHOLD 0x68 #define INTC_MIR_CLEAR(x) (0x88 + ((x) * 0x20)) #define INTC_MIR_SET(x) (0x8C + ((x) * 0x20)) #define INTC_ISR_SET(x) (0x90 + ((x) * 0x20)) #define INTC_ISR_CLEAR(x) (0x94 + ((x) * 0x20)) #define INTC_SIR_SPURIOUS_MASK 0xffffff80 #define INTC_SIR_ACTIVE_MASK 0x7f #define INTC_NIRQS 128 struct ti_aintc_irqsrc { struct intr_irqsrc tai_isrc; u_int tai_irq; }; struct ti_aintc_softc { device_t sc_dev; struct resource * aintc_res[3]; bus_space_tag_t aintc_bst; bus_space_handle_t aintc_bsh; uint8_t ver; struct ti_aintc_irqsrc aintc_isrcs[INTC_NIRQS]; }; static struct resource_spec ti_aintc_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; #define aintc_read_4(_sc, reg) \ bus_space_read_4((_sc)->aintc_bst, (_sc)->aintc_bsh, (reg)) #define aintc_write_4(_sc, reg, val) \ bus_space_write_4((_sc)->aintc_bst, (_sc)->aintc_bsh, (reg), (val)) /* List of compatible strings for FDT tree */ static struct ofw_compat_data compat_data[] = { {"ti,am33xx-intc", 1}, {"ti,omap2-intc", 1}, {NULL, 0}, }; static inline void ti_aintc_irq_eoi(struct ti_aintc_softc *sc) { aintc_write_4(sc, INTC_CONTROL, 1); } static inline void ti_aintc_irq_mask(struct ti_aintc_softc *sc, u_int irq) { aintc_write_4(sc, INTC_MIR_SET(irq >> 5), (1UL << (irq & 0x1F))); } static inline void ti_aintc_irq_unmask(struct ti_aintc_softc *sc, u_int irq) { aintc_write_4(sc, INTC_MIR_CLEAR(irq >> 5), (1UL << (irq & 0x1F))); } static int ti_aintc_intr(void *arg) { uint32_t irq; struct ti_aintc_softc *sc = arg; /* Get active interrupt */ irq = aintc_read_4(sc, INTC_SIR_IRQ); if ((irq & INTC_SIR_SPURIOUS_MASK) != 0) { device_printf(sc->sc_dev, "Spurious interrupt detected (0x%08x)\n", irq); ti_aintc_irq_eoi(sc); return (FILTER_HANDLED); } /* Only level-sensitive interrupts detection is supported. */ irq &= INTC_SIR_ACTIVE_MASK; if (intr_isrc_dispatch(&sc->aintc_isrcs[irq].tai_isrc, curthread->td_intr_frame) != 0) { ti_aintc_irq_mask(sc, irq); ti_aintc_irq_eoi(sc); device_printf(sc->sc_dev, "Stray irq %u disabled\n", irq); } arm_irq_memory_barrier(irq); /* XXX */ return (FILTER_HANDLED); } static void ti_aintc_enable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq = ((struct ti_aintc_irqsrc *)isrc)->tai_irq; struct ti_aintc_softc *sc = device_get_softc(dev); arm_irq_memory_barrier(irq); ti_aintc_irq_unmask(sc, irq); } static void ti_aintc_disable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq = ((struct ti_aintc_irqsrc *)isrc)->tai_irq; struct ti_aintc_softc *sc = device_get_softc(dev); ti_aintc_irq_mask(sc, irq); } static int ti_aintc_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct intr_map_data_fdt *daf; struct ti_aintc_softc *sc; if (data->type != INTR_MAP_DATA_FDT) return (ENOTSUP); daf = (struct intr_map_data_fdt *)data; if (daf->ncells != 1 || daf->cells[0] >= INTC_NIRQS) return (EINVAL); sc = device_get_softc(dev); *isrcp = &sc->aintc_isrcs[daf->cells[0]].tai_isrc; return (0); } static void ti_aintc_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { u_int irq = ((struct ti_aintc_irqsrc *)isrc)->tai_irq; struct ti_aintc_softc *sc = device_get_softc(dev); ti_aintc_irq_mask(sc, irq); ti_aintc_irq_eoi(sc); } static void ti_aintc_post_ithread(device_t dev, struct intr_irqsrc *isrc) { ti_aintc_enable_intr(dev, isrc); } static void ti_aintc_post_filter(device_t dev, struct intr_irqsrc *isrc) { ti_aintc_irq_eoi(device_get_softc(dev)); } static int ti_aintc_pic_attach(struct ti_aintc_softc *sc) { struct intr_pic *pic; int error; uint32_t irq; const char *name; intptr_t xref; name = device_get_nameunit(sc->sc_dev); for (irq = 0; irq < INTC_NIRQS; irq++) { sc->aintc_isrcs[irq].tai_irq = irq; error = intr_isrc_register(&sc->aintc_isrcs[irq].tai_isrc, sc->sc_dev, 0, "%s,%u", name, irq); if (error != 0) return (error); } xref = OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)); pic = intr_pic_register(sc->sc_dev, xref); if (pic == NULL) return (ENXIO); - return (intr_pic_claim_root(sc->sc_dev, xref, ti_aintc_intr, sc, 0)); + return (intr_pic_claim_root(sc->sc_dev, xref, ti_aintc_intr, sc)); } static int ti_aintc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) return (ENXIO); device_set_desc(dev, "TI AINTC Interrupt Controller"); return (BUS_PROBE_DEFAULT); } static int ti_aintc_attach(device_t dev) { struct ti_aintc_softc *sc = device_get_softc(dev); uint32_t x; sc->sc_dev = dev; if (bus_alloc_resources(dev, ti_aintc_spec, sc->aintc_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->aintc_bst = rman_get_bustag(sc->aintc_res[0]); sc->aintc_bsh = rman_get_bushandle(sc->aintc_res[0]); x = aintc_read_4(sc, INTC_REVISION); device_printf(dev, "Revision %u.%u\n",(x >> 4) & 0xF, x & 0xF); /* SoftReset */ aintc_write_4(sc, INTC_SYSCONFIG, 2); /* Wait for reset to complete */ while(!(aintc_read_4(sc, INTC_SYSSTATUS) & 1)); /*Set Priority Threshold */ aintc_write_4(sc, INTC_THRESHOLD, 0xFF); if (ti_aintc_pic_attach(sc) != 0) { device_printf(dev, "could not attach PIC\n"); return (ENXIO); } return (0); } static device_method_t ti_aintc_methods[] = { DEVMETHOD(device_probe, ti_aintc_probe), DEVMETHOD(device_attach, ti_aintc_attach), DEVMETHOD(pic_disable_intr, ti_aintc_disable_intr), DEVMETHOD(pic_enable_intr, ti_aintc_enable_intr), DEVMETHOD(pic_map_intr, ti_aintc_map_intr), DEVMETHOD(pic_post_filter, ti_aintc_post_filter), DEVMETHOD(pic_post_ithread, ti_aintc_post_ithread), DEVMETHOD(pic_pre_ithread, ti_aintc_pre_ithread), { 0, 0 } }; static driver_t ti_aintc_driver = { "ti_aintc", ti_aintc_methods, sizeof(struct ti_aintc_softc), }; EARLY_DRIVER_MODULE(ti_aintc, simplebus, ti_aintc_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); SIMPLEBUS_PNP_INFO(compat_data); diff --git a/sys/arm64/arm64/gic_v3_acpi.c b/sys/arm64/arm64/gic_v3_acpi.c index 3dff49f6214f..e55aea86157e 100644 --- a/sys/arm64/arm64/gic_v3_acpi.c +++ b/sys/arm64/arm64/gic_v3_acpi.c @@ -1,470 +1,470 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation * Copyright (c) 2022 Arm Ltd * * This software was developed by Andrew Turner under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_acpi.h" #include #include #include #include #include #include #include #include #include #include #include #include "gic_v3_reg.h" #include "gic_v3_var.h" #define GICV3_PRIV_VGIC 0x80000000 #define GICV3_PRIV_FLAGS 0x80000000 #define HV_MSI_SPI_START 64 #define HV_MSI_SPI_LAST 0 struct gic_v3_acpi_devinfo { struct gic_v3_devinfo di_gic_dinfo; struct resource_list di_rl; }; static device_identify_t gic_v3_acpi_identify; static device_probe_t gic_v3_acpi_probe; static device_attach_t gic_v3_acpi_attach; static bus_get_resource_list_t gic_v3_acpi_get_resource_list; static void gic_v3_acpi_bus_attach(device_t); static device_method_t gic_v3_acpi_methods[] = { /* Device interface */ DEVMETHOD(device_identify, gic_v3_acpi_identify), DEVMETHOD(device_probe, gic_v3_acpi_probe), DEVMETHOD(device_attach, gic_v3_acpi_attach), /* Bus interface */ DEVMETHOD(bus_get_resource_list, gic_v3_acpi_get_resource_list), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(gic, gic_v3_acpi_driver, gic_v3_acpi_methods, sizeof(struct gic_v3_softc), gic_v3_driver); EARLY_DRIVER_MODULE(gic_v3, acpi, gic_v3_acpi_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); struct madt_table_data { device_t parent; device_t dev; ACPI_MADT_GENERIC_DISTRIBUTOR *dist; int count; bool rdist_use_gicc; bool have_vgic; }; static void madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct madt_table_data *madt_data; madt_data = (struct madt_table_data *)arg; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR: if (madt_data->dist != NULL) { if (bootverbose) device_printf(madt_data->parent, "gic: Already have a distributor table"); break; } madt_data->dist = (ACPI_MADT_GENERIC_DISTRIBUTOR *)entry; break; case ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR: break; default: break; } } static void rdist_map(ACPI_SUBTABLE_HEADER *entry, void *arg) { ACPI_MADT_GENERIC_REDISTRIBUTOR *redist; ACPI_MADT_GENERIC_INTERRUPT *intr; struct madt_table_data *madt_data; rman_res_t count; madt_data = (struct madt_table_data *)arg; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR: if (madt_data->rdist_use_gicc) break; redist = (ACPI_MADT_GENERIC_REDISTRIBUTOR *)entry; madt_data->count++; BUS_SET_RESOURCE(madt_data->parent, madt_data->dev, SYS_RES_MEMORY, madt_data->count, redist->BaseAddress, redist->Length); break; case ACPI_MADT_TYPE_GENERIC_INTERRUPT: if (!madt_data->rdist_use_gicc) break; intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; madt_data->count++; /* * Map the two 64k redistributor frames. */ count = GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE; if (madt_data->dist->Version == ACPI_MADT_GIC_VERSION_V4) count += GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE; BUS_SET_RESOURCE(madt_data->parent, madt_data->dev, SYS_RES_MEMORY, madt_data->count, intr->GicrBaseAddress, count); if (intr->VgicInterrupt == 0) madt_data->have_vgic = false; default: break; } } static void gic_v3_acpi_identify(driver_t *driver, device_t parent) { struct madt_table_data madt_data; ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; uintptr_t private; device_t dev; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return; madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(parent, "gic: Unable to map the MADT\n"); return; } madt_data.parent = parent; madt_data.dist = NULL; madt_data.count = 0; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_handler, &madt_data); if (madt_data.dist == NULL) { device_printf(parent, "No gic interrupt or distributor table\n"); goto out; } /* Check the GIC version is supported by thiss driver */ switch(madt_data.dist->Version) { case ACPI_MADT_GIC_VERSION_V3: case ACPI_MADT_GIC_VERSION_V4: break; default: goto out; } dev = BUS_ADD_CHILD(parent, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE, "gic", -1); if (dev == NULL) { device_printf(parent, "add gic child failed\n"); goto out; } /* Add the MADT data */ BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 0, madt_data.dist->BaseAddress, GICD_SIZE); madt_data.dev = dev; madt_data.rdist_use_gicc = false; madt_data.have_vgic = true; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, rdist_map, &madt_data); if (madt_data.count == 0) { /* * No redistributors found, fall back to use the GICR * address from the GICC sub-table. */ madt_data.rdist_use_gicc = true; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, rdist_map, &madt_data); } private = madt_data.dist->Version; /* Flag that the VGIC is in use */ if (madt_data.have_vgic) private |= GICV3_PRIV_VGIC; acpi_set_private(dev, (void *)private); out: acpi_unmap_table(madt); } static int gic_v3_acpi_probe(device_t dev) { switch((uintptr_t)acpi_get_private(dev) & ~GICV3_PRIV_FLAGS) { case ACPI_MADT_GIC_VERSION_V3: case ACPI_MADT_GIC_VERSION_V4: break; default: return (ENXIO); } device_set_desc(dev, GIC_V3_DEVSTR); return (BUS_PROBE_NOWILDCARD); } static void madt_count_redistrib(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct gic_v3_softc *sc = arg; if (entry->Type == ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR) sc->gic_redists.nregions++; } static void madt_count_gicc_redistrib(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct gic_v3_softc *sc = arg; if (entry->Type == ACPI_MADT_TYPE_GENERIC_INTERRUPT) sc->gic_redists.nregions++; } static int gic_v3_acpi_count_regions(device_t dev) { struct gic_v3_softc *sc; ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; sc = device_get_softc(dev); physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return (ENXIO); madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(dev, "Unable to map the MADT\n"); return (ENXIO); } acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_count_redistrib, sc); /* Fall back to use the distributor GICR base address */ if (sc->gic_redists.nregions == 0) { acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_count_gicc_redistrib, sc); } acpi_unmap_table(madt); return (sc->gic_redists.nregions > 0 ? 0 : ENXIO); } static int gic_v3_acpi_attach(device_t dev) { struct gic_v3_softc *sc; int err; sc = device_get_softc(dev); sc->dev = dev; sc->gic_bus = GIC_BUS_ACPI; err = gic_v3_acpi_count_regions(dev); if (err != 0) goto count_error; if (vm_guest == VM_GUEST_HV) { sc->gic_mbi_start = HV_MSI_SPI_START; sc->gic_mbi_end = HV_MSI_SPI_LAST; } err = gic_v3_attach(dev); if (err != 0) goto error; sc->gic_pic = intr_pic_register(dev, ACPI_INTR_XREF); if (sc->gic_pic == NULL) { device_printf(dev, "could not register PIC\n"); err = ENXIO; goto error; } /* * Registering for MSI with SPI range, as this is * required for Hyper-V GIC to work in ARM64. */ if (vm_guest == VM_GUEST_HV) { err = intr_msi_register(dev, ACPI_MSI_XREF); if (err) { device_printf(dev, "could not register MSI\n"); goto error; } } - if (intr_pic_claim_root(dev, ACPI_INTR_XREF, arm_gic_v3_intr, sc, - GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { + if (intr_pic_claim_root(dev, ACPI_INTR_XREF, arm_gic_v3_intr, sc) + != 0) { err = ENXIO; goto error; } /* * Try to register the ITS driver to this GIC. The GIC will act as * a bus in that case. Failure here will not affect the main GIC * functionality. */ gic_v3_acpi_bus_attach(dev); if (device_get_children(dev, &sc->gic_children, &sc->gic_nchildren) !=0) sc->gic_nchildren = 0; return (0); error: /* Failure so free resources */ gic_v3_detach(dev); count_error: if (bootverbose) { device_printf(dev, "Failed to attach. Error %d\n", err); } return (err); } static void gic_v3_add_children(ACPI_SUBTABLE_HEADER *entry, void *arg) { ACPI_MADT_GENERIC_TRANSLATOR *gict; struct gic_v3_acpi_devinfo *di; struct gic_v3_softc *sc; device_t child, dev; u_int xref; int err, pxm; if (entry->Type == ACPI_MADT_TYPE_GENERIC_TRANSLATOR) { /* We have an ITS, add it as a child */ gict = (ACPI_MADT_GENERIC_TRANSLATOR *)entry; dev = arg; sc = device_get_softc(dev); di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); err = acpi_iort_its_lookup(gict->TranslationId, &xref, &pxm); if (err != 0) { free(di, M_GIC_V3); return; } child = device_add_child(dev, "its", -1); if (child == NULL) { free(di, M_GIC_V3); return; } di->di_gic_dinfo.gic_domain = pxm; di->di_gic_dinfo.msi_xref = xref; resource_list_init(&di->di_rl); resource_list_add(&di->di_rl, SYS_RES_MEMORY, 0, gict->BaseAddress, gict->BaseAddress + 128 * 1024 - 1, 128 * 1024); sc->gic_nchildren++; device_set_ivars(child, di); } } static void gic_v3_acpi_bus_attach(device_t dev) { struct gic_v3_acpi_devinfo *di; struct gic_v3_softc *sc; ACPI_TABLE_MADT *madt; device_t child; vm_paddr_t physaddr; sc = device_get_softc(dev); physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return; madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(dev, "Unable to map the MADT to add children\n"); return; } acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, gic_v3_add_children, dev); /* Add the vgic child if needed */ if (((uintptr_t)acpi_get_private(dev) & GICV3_PRIV_FLAGS) != 0) { child = device_add_child(dev, "vgic", -1); if (child == NULL) { device_printf(dev, "Could not add vgic child\n"); } else { di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); resource_list_init(&di->di_rl); di->di_gic_dinfo.gic_domain = -1; di->di_gic_dinfo.is_vgic = 1; device_set_ivars(child, di); sc->gic_nchildren++; } } acpi_unmap_table(madt); bus_generic_attach(dev); } static struct resource_list * gic_v3_acpi_get_resource_list(device_t bus, device_t child) { struct gic_v3_acpi_devinfo *di; di = device_get_ivars(child); KASSERT(di != NULL, ("%s: No devinfo", __func__)); return (&di->di_rl); } diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c index e48e7b36a649..8adb0497771e 100644 --- a/sys/arm64/arm64/gic_v3_fdt.c +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -1,383 +1,382 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gic_v3_reg.h" #include "gic_v3_var.h" /* * FDT glue. */ static int gic_v3_fdt_probe(device_t); static int gic_v3_fdt_attach(device_t); static const struct ofw_bus_devinfo *gic_v3_ofw_get_devinfo(device_t, device_t); static bus_get_resource_list_t gic_v3_fdt_get_resource_list; static device_method_t gic_v3_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gic_v3_fdt_probe), DEVMETHOD(device_attach, gic_v3_fdt_attach), /* Bus interface */ DEVMETHOD(bus_get_resource_list, gic_v3_fdt_get_resource_list), DEVMETHOD(bus_get_device_path, ofw_bus_gen_get_device_path), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, gic_v3_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(gic, gic_v3_fdt_driver, gic_v3_fdt_methods, sizeof(struct gic_v3_softc), gic_v3_driver); EARLY_DRIVER_MODULE(gic_v3, simplebus, gic_v3_fdt_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic_v3, ofwbus, gic_v3_fdt_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); /* * Helper functions declarations. */ static int gic_v3_ofw_bus_attach(device_t); /* * Device interface. */ static int gic_v3_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "arm,gic-v3")) return (ENXIO); device_set_desc(dev, GIC_V3_DEVSTR); return (BUS_PROBE_DEFAULT); } static int gic_v3_fdt_attach(device_t dev) { struct gic_v3_softc *sc; pcell_t redist_regions; intptr_t xref; int err; uint32_t *mbi_ranges; ssize_t ret; sc = device_get_softc(dev); sc->dev = dev; sc->gic_bus = GIC_BUS_FDT; /* * Recover number of the Re-Distributor regions. */ if (OF_getencprop(ofw_bus_get_node(dev), "#redistributor-regions", &redist_regions, sizeof(redist_regions)) <= 0) sc->gic_redists.nregions = 1; else sc->gic_redists.nregions = redist_regions; /* Add Message Based Interrupts using SPIs. */ ret = OF_getencprop_alloc_multi(ofw_bus_get_node(dev), "mbi-ranges", sizeof(*mbi_ranges), (void **)&mbi_ranges); if (ret > 0) { if (ret % 2 == 0) { /* Limit to a single range for now. */ sc->gic_mbi_start = mbi_ranges[0]; sc->gic_mbi_end = mbi_ranges[0] + mbi_ranges[1]; } else { if (bootverbose) device_printf(dev, "Malformed mbi-ranges property\n"); } free(mbi_ranges, M_OFWPROP); } err = gic_v3_attach(dev); if (err != 0) goto error; xref = OF_xref_from_node(ofw_bus_get_node(dev)); sc->gic_pic = intr_pic_register(dev, xref); if (sc->gic_pic == NULL) { device_printf(dev, "could not register PIC\n"); err = ENXIO; goto error; } if (sc->gic_mbi_start > 0) intr_msi_register(dev, xref); /* Register xref */ OF_device_register_xref(xref, dev); - if (intr_pic_claim_root(dev, xref, arm_gic_v3_intr, sc, - GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { + if (intr_pic_claim_root(dev, xref, arm_gic_v3_intr, sc) != 0) { err = ENXIO; goto error; } /* * Try to register ITS to this GIC. * GIC will act as a bus in that case. * Failure here will not affect main GIC functionality. */ if (gic_v3_ofw_bus_attach(dev) != 0) { if (bootverbose) { device_printf(dev, "Failed to attach ITS to this GIC\n"); } } if (device_get_children(dev, &sc->gic_children, &sc->gic_nchildren) != 0) sc->gic_nchildren = 0; return (err); error: if (bootverbose) { device_printf(dev, "Failed to attach. Error %d\n", err); } /* Failure so free resources */ gic_v3_detach(dev); return (err); } /* OFW bus interface */ struct gic_v3_ofw_devinfo { struct gic_v3_devinfo di_gic_dinfo; struct ofw_bus_devinfo di_dinfo; struct resource_list di_rl; }; static const struct ofw_bus_devinfo * gic_v3_ofw_get_devinfo(device_t bus __unused, device_t child) { struct gic_v3_ofw_devinfo *di; di = device_get_ivars(child); if (di->di_gic_dinfo.is_vgic) return (NULL); return (&di->di_dinfo); } /* Helper functions */ static int gic_v3_ofw_fill_ranges(phandle_t parent, struct gic_v3_softc *sc, pcell_t *addr_cellsp, pcell_t *size_cellsp) { pcell_t addr_cells, host_cells, size_cells; cell_t *base_ranges; ssize_t nbase_ranges; int i, j, k; host_cells = 1; OF_getencprop(OF_parent(parent), "#address-cells", &host_cells, sizeof(host_cells)); addr_cells = 2; OF_getencprop(parent, "#address-cells", &addr_cells, sizeof(addr_cells)); size_cells = 2; OF_getencprop(parent, "#size-cells", &size_cells, sizeof(size_cells)); *addr_cellsp = addr_cells; *size_cellsp = size_cells; nbase_ranges = OF_getproplen(parent, "ranges"); if (nbase_ranges < 0) return (EINVAL); sc->nranges = nbase_ranges / sizeof(cell_t) / (addr_cells + host_cells + size_cells); if (sc->nranges == 0) return (0); sc->ranges = malloc(sc->nranges * sizeof(sc->ranges[0]), M_GIC_V3, M_WAITOK); base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); OF_getencprop(parent, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->nranges; i++) { sc->ranges[i].bus = 0; for (k = 0; k < addr_cells; k++) { sc->ranges[i].bus <<= 32; sc->ranges[i].bus |= base_ranges[j++]; } sc->ranges[i].host = 0; for (k = 0; k < host_cells; k++) { sc->ranges[i].host <<= 32; sc->ranges[i].host |= base_ranges[j++]; } sc->ranges[i].size = 0; for (k = 0; k < size_cells; k++) { sc->ranges[i].size <<= 32; sc->ranges[i].size |= base_ranges[j++]; } } free(base_ranges, M_DEVBUF); return (0); } /* * Bus capability support for GICv3. * Collects and configures device informations and finally * adds ITS device as a child of GICv3 in Newbus hierarchy. */ static int gic_v3_ofw_bus_attach(device_t dev) { struct gic_v3_ofw_devinfo *di; struct gic_v3_softc *sc; device_t child; phandle_t parent, node; pcell_t addr_cells, size_cells; int rv; sc = device_get_softc(dev); parent = ofw_bus_get_node(dev); if (parent > 0) { rv = gic_v3_ofw_fill_ranges(parent, sc, &addr_cells, &size_cells); if (rv != 0) return (rv); /* Iterate through all GIC subordinates */ for (node = OF_child(parent); node > 0; node = OF_peer(node)) { /* * Ignore children that lack a compatible property. * Some of them may be for configuration, for example * ppi-partitions. */ if (!OF_hasprop(node, "compatible")) continue; /* Allocate and populate devinfo. */ di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); /* Read the numa node, or -1 if there is none */ if (OF_getencprop(node, "numa-node-id", &di->di_gic_dinfo.gic_domain, sizeof(di->di_gic_dinfo.gic_domain)) <= 0) { di->di_gic_dinfo.gic_domain = -1; } if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node)) { if (bootverbose) { device_printf(dev, "Could not set up devinfo for ITS\n"); } free(di, M_GIC_V3); continue; } /* Initialize and populate resource list. */ resource_list_init(&di->di_rl); ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells, &di->di_rl); /* Should not have any interrupts, so don't add any */ /* Add newbus device for this FDT node */ child = device_add_child(dev, NULL, -1); if (!child) { if (bootverbose) { device_printf(dev, "Could not add child: %s\n", di->di_dinfo.obd_name); } resource_list_free(&di->di_rl); ofw_bus_gen_destroy_devinfo(&di->di_dinfo); free(di, M_GIC_V3); continue; } sc->gic_nchildren++; device_set_ivars(child, di); } } /* * If there is a vgic maintanance interrupt add a virtual gic * child so we can use this in the vmm module for bhyve. */ if (OF_hasprop(parent, "interrupts")) { child = device_add_child(dev, "vgic", -1); if (child == NULL) { device_printf(dev, "Could not add vgic child\n"); } else { di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); resource_list_init(&di->di_rl); di->di_gic_dinfo.gic_domain = -1; di->di_gic_dinfo.is_vgic = 1; device_set_ivars(child, di); sc->gic_nchildren++; } } return (bus_generic_attach(dev)); } static struct resource_list * gic_v3_fdt_get_resource_list(device_t bus, device_t child) { struct gic_v3_ofw_devinfo *di; di = device_get_ivars(child); KASSERT(di != NULL, ("%s: No devinfo", __func__)); return (&di->di_rl); } diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c index 49fe20cdc890..fb75b2e1773b 100644 --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -1,1778 +1,1776 @@ /*- * Copyright (c) 2015-2016 Svatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * New-style Interrupt Framework * * TODO: - add support for disconnected PICs. * - to support IPI (PPI) enabling on other CPUs if already started. * - to complete things for removable PICs. */ #include "opt_ddb.h" #include "opt_hwpmc_hooks.h" #include "opt_iommu.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HWPMC_HOOKS #include #endif #include #include #include #include #include #ifdef DDB #include #endif #ifdef IOMMU #include #endif #include "pic_if.h" #include "msi_if.h" #define INTRNAME_LEN (2*MAXCOMLEN + 1) #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif MALLOC_DECLARE(M_INTRNG); MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); /* Main interrupt handler called from assembler -> 'hidden' for C code. */ void intr_irq_handler(struct trapframe *tf); /* Root interrupt controller stuff. */ device_t intr_irq_root_dev; static intr_irq_filter_t *irq_root_filter; static void *irq_root_arg; -static u_int irq_root_ipicount; struct intr_pic_child { SLIST_ENTRY(intr_pic_child) pc_next; struct intr_pic *pc_pic; intr_child_irq_filter_t *pc_filter; void *pc_filter_arg; uintptr_t pc_start; uintptr_t pc_length; }; /* Interrupt controller definition. */ struct intr_pic { SLIST_ENTRY(intr_pic) pic_next; intptr_t pic_xref; /* hardware identification */ device_t pic_dev; /* Only one of FLAG_PIC or FLAG_MSI may be set */ #define FLAG_PIC (1 << 0) #define FLAG_MSI (1 << 1) #define FLAG_TYPE_MASK (FLAG_PIC | FLAG_MSI) u_int pic_flags; struct mtx pic_child_lock; SLIST_HEAD(, intr_pic_child) pic_children; }; static struct mtx pic_list_lock; static SLIST_HEAD(, intr_pic) pic_list; static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, int flags); /* Interrupt source definition. */ static struct mtx isrc_table_lock; static struct intr_irqsrc **irq_sources; static u_int irq_next_free; #ifdef SMP #ifdef EARLY_AP_STARTUP static bool irq_assign_cpu = true; #else static bool irq_assign_cpu = false; #endif #endif u_int intr_nirq = NIRQ; SYSCTL_UINT(_machdep, OID_AUTO, nirq, CTLFLAG_RDTUN, &intr_nirq, 0, "Number of IRQs"); /* Data for MI statistics reporting. */ u_long *intrcnt; char *intrnames; size_t sintrcnt; size_t sintrnames; int nintrcnt; static bitstr_t *intrcnt_bitmap; static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); static struct intr_map_data * intr_map_get_map_data(u_int res_id); static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, struct intr_map_data **data); /* * Interrupt framework initialization routine. */ static void intr_irq_init(void *dummy __unused) { SLIST_INIT(&pic_list); mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); /* * - 2 counters for each I/O interrupt. * - mp_maxid + 1 counters for each IPI counters for SMP. */ nintrcnt = intr_nirq * 2; #ifdef SMP nintrcnt += INTR_IPI_COUNT * (mp_maxid + 1); #endif intrcnt = mallocarray(nintrcnt, sizeof(u_long), M_INTRNG, M_WAITOK | M_ZERO); intrnames = mallocarray(nintrcnt, INTRNAME_LEN, M_INTRNG, M_WAITOK | M_ZERO); sintrcnt = nintrcnt * sizeof(u_long); sintrnames = nintrcnt * INTRNAME_LEN; /* Allocate the bitmap tracking counter allocations. */ intrcnt_bitmap = bit_alloc(nintrcnt, M_INTRNG, M_WAITOK | M_ZERO); irq_sources = mallocarray(intr_nirq, sizeof(struct intr_irqsrc*), M_INTRNG, M_WAITOK | M_ZERO); } SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); static void intrcnt_setname(const char *name, int index) { snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", INTRNAME_LEN - 1, name); } /* * Update name for interrupt source with interrupt event. */ static void intrcnt_updatename(struct intr_irqsrc *isrc) { /* QQQ: What about stray counter name? */ mtx_assert(&isrc_table_lock, MA_OWNED); intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); } /* * Virtualization for interrupt source interrupt counter increment. */ static inline void isrc_increment_count(struct intr_irqsrc *isrc) { if (isrc->isrc_flags & INTR_ISRCF_PPI) atomic_add_long(&isrc->isrc_count[0], 1); else isrc->isrc_count[0]++; } /* * Virtualization for interrupt source interrupt stray counter increment. */ static inline void isrc_increment_straycount(struct intr_irqsrc *isrc) { isrc->isrc_count[1]++; } /* * Virtualization for interrupt source interrupt name update. */ static void isrc_update_name(struct intr_irqsrc *isrc, const char *name) { char str[INTRNAME_LEN]; mtx_assert(&isrc_table_lock, MA_OWNED); if (name != NULL) { snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); intrcnt_setname(str, isrc->isrc_index); snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, name); intrcnt_setname(str, isrc->isrc_index + 1); } else { snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); intrcnt_setname(str, isrc->isrc_index); snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); intrcnt_setname(str, isrc->isrc_index + 1); } } /* * Virtualization for interrupt source interrupt counters setup. */ static void isrc_setup_counters(struct intr_irqsrc *isrc) { int index; mtx_assert(&isrc_table_lock, MA_OWNED); /* * Allocate two counter values, the second tracking "stray" interrupts. */ bit_ffc_area(intrcnt_bitmap, nintrcnt, 2, &index); if (index == -1) panic("Failed to allocate 2 counters. Array exhausted?"); bit_nset(intrcnt_bitmap, index, index + 1); isrc->isrc_index = index; isrc->isrc_count = &intrcnt[index]; isrc_update_name(isrc, NULL); } /* * Virtualization for interrupt source interrupt counters release. */ static void isrc_release_counters(struct intr_irqsrc *isrc) { int idx = isrc->isrc_index; mtx_assert(&isrc_table_lock, MA_OWNED); bit_nclear(intrcnt_bitmap, idx, idx + 1); } #ifdef SMP /* * Virtualization for interrupt source IPI counters setup. */ u_long * intr_ipi_setup_counters(const char *name) { u_int index, i; char str[INTRNAME_LEN]; mtx_lock(&isrc_table_lock); /* * We should never have a problem finding mp_maxid + 1 contiguous * counters, in practice. Interrupts will be allocated sequentially * during boot, so the array should fill from low to high index. Once * reserved, the IPI counters will never be released. Similarly, we * will not need to allocate more IPIs once the system is running. */ bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index); if (index == -1) panic("Failed to allocate %d counters. Array exhausted?", mp_maxid + 1); bit_nset(intrcnt_bitmap, index, index + mp_maxid); for (i = 0; i < mp_maxid + 1; i++) { snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); intrcnt_setname(str, index + i); } mtx_unlock(&isrc_table_lock); return (&intrcnt[index]); } #endif /* * Main interrupt dispatch handler. It's called straight * from the assembler, where CPU interrupt is served. */ void intr_irq_handler(struct trapframe *tf) { struct trapframe * oldframe; struct thread * td; KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); kasan_mark(tf, sizeof(*tf), sizeof(*tf), 0); VM_CNT_INC(v_intr); critical_enter(); td = curthread; oldframe = td->td_intr_frame; td->td_intr_frame = tf; irq_root_filter(irq_root_arg); td->td_intr_frame = oldframe; critical_exit(); #ifdef HWPMC_HOOKS if (pmc_hook && TRAPF_USERMODE(tf) && (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); #endif } int intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) { struct intr_pic_child *child; bool found; found = false; mtx_lock_spin(&parent->pic_child_lock); SLIST_FOREACH(child, &parent->pic_children, pc_next) { if (child->pc_start <= irq && irq < (child->pc_start + child->pc_length)) { found = true; break; } } mtx_unlock_spin(&parent->pic_child_lock); if (found) return (child->pc_filter(child->pc_filter_arg, irq)); return (FILTER_STRAY); } /* * interrupt controller dispatch function for interrupts. It should * be called straight from the interrupt controller, when associated interrupt * source is learned. */ int intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) { KASSERT(isrc != NULL, ("%s: no source", __func__)); isrc_increment_count(isrc); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) { int error; error = isrc->isrc_filter(isrc->isrc_arg, tf); PIC_POST_FILTER(isrc->isrc_dev, isrc); if (error == FILTER_HANDLED) return (0); } else #endif if (isrc->isrc_event != NULL) { if (intr_event_handle(isrc->isrc_event, tf) == 0) return (0); } isrc_increment_straycount(isrc); return (EINVAL); } /* * Alloc unique interrupt number (resource handle) for interrupt source. * * There could be various strategies how to allocate free interrupt number * (resource handle) for new interrupt source. * * 1. Handles are always allocated forward, so handles are not recycled * immediately. However, if only one free handle left which is reused * constantly... */ static inline int isrc_alloc_irq(struct intr_irqsrc *isrc) { u_int irq; mtx_assert(&isrc_table_lock, MA_OWNED); if (irq_next_free >= intr_nirq) return (ENOSPC); for (irq = irq_next_free; irq < intr_nirq; irq++) { if (irq_sources[irq] == NULL) goto found; } for (irq = 0; irq < irq_next_free; irq++) { if (irq_sources[irq] == NULL) goto found; } irq_next_free = intr_nirq; return (ENOSPC); found: isrc->isrc_irq = irq; irq_sources[irq] = isrc; irq_next_free = irq + 1; if (irq_next_free >= intr_nirq) irq_next_free = 0; return (0); } /* * Free unique interrupt number (resource handle) from interrupt source. */ static inline int isrc_free_irq(struct intr_irqsrc *isrc) { mtx_assert(&isrc_table_lock, MA_OWNED); if (isrc->isrc_irq >= intr_nirq) return (EINVAL); if (irq_sources[isrc->isrc_irq] != isrc) return (EINVAL); irq_sources[isrc->isrc_irq] = NULL; isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ /* * If we are recovering from the state irq_sources table is full, * then the following allocation should check the entire table. This * will ensure maximum separation of allocation order from release * order. */ if (irq_next_free >= intr_nirq) irq_next_free = 0; return (0); } /* * Initialize interrupt source and register it into global interrupt table. */ int intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, const char *fmt, ...) { int error; va_list ap; bzero(isrc, sizeof(struct intr_irqsrc)); isrc->isrc_dev = dev; isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ isrc->isrc_flags = flags; va_start(ap, fmt); vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); va_end(ap); mtx_lock(&isrc_table_lock); error = isrc_alloc_irq(isrc); if (error != 0) { mtx_unlock(&isrc_table_lock); return (error); } /* * Setup interrupt counters, but not for IPI sources. Those are setup * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust * our counter pool. */ if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) isrc_setup_counters(isrc); mtx_unlock(&isrc_table_lock); return (0); } /* * Deregister interrupt source from global interrupt table. */ int intr_isrc_deregister(struct intr_irqsrc *isrc) { int error; mtx_lock(&isrc_table_lock); if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) isrc_release_counters(isrc); error = isrc_free_irq(isrc); mtx_unlock(&isrc_table_lock); return (error); } #ifdef SMP /* * A support function for a PIC to decide if provided ISRC should be inited * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of * struct intr_irqsrc is the following: * * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus * set in isrc_cpu. If not, the ISRC should be inited on every cpu and * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. */ bool intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) { if (isrc->isrc_handlers == 0) return (false); if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) return (false); if (isrc->isrc_flags & INTR_ISRCF_BOUND) return (CPU_ISSET(cpu, &isrc->isrc_cpu)); CPU_SET(cpu, &isrc->isrc_cpu); return (true); } #endif #ifdef INTR_SOLO /* * Setup filter into interrupt source. */ static int iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, intr_irq_filter_t *filter, void *arg, void **cookiep) { if (filter == NULL) return (EINVAL); mtx_lock(&isrc_table_lock); /* * Make sure that we do not mix the two ways * how we handle interrupt sources. */ if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { mtx_unlock(&isrc_table_lock); return (EBUSY); } isrc->isrc_filter = filter; isrc->isrc_arg = arg; isrc_update_name(isrc, name); mtx_unlock(&isrc_table_lock); *cookiep = isrc; return (0); } #endif /* * Interrupt source pre_ithread method for MI interrupt framework. */ static void intr_isrc_pre_ithread(void *arg) { struct intr_irqsrc *isrc = arg; PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); } /* * Interrupt source post_ithread method for MI interrupt framework. */ static void intr_isrc_post_ithread(void *arg) { struct intr_irqsrc *isrc = arg; PIC_POST_ITHREAD(isrc->isrc_dev, isrc); } /* * Interrupt source post_filter method for MI interrupt framework. */ static void intr_isrc_post_filter(void *arg) { struct intr_irqsrc *isrc = arg; PIC_POST_FILTER(isrc->isrc_dev, isrc); } /* * Interrupt source assign_cpu method for MI interrupt framework. */ static int intr_isrc_assign_cpu(void *arg, int cpu) { #ifdef SMP struct intr_irqsrc *isrc = arg; int error; mtx_lock(&isrc_table_lock); if (cpu == NOCPU) { CPU_ZERO(&isrc->isrc_cpu); isrc->isrc_flags &= ~INTR_ISRCF_BOUND; } else { CPU_SETOF(cpu, &isrc->isrc_cpu); isrc->isrc_flags |= INTR_ISRCF_BOUND; } /* * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or * re-balance it to another CPU or enable it on more CPUs. However, * PIC is expected to change isrc_cpu appropriately to keep us well * informed if the call is successful. */ if (irq_assign_cpu) { error = PIC_BIND_INTR(isrc->isrc_dev, isrc); if (error) { CPU_ZERO(&isrc->isrc_cpu); mtx_unlock(&isrc_table_lock); return (error); } } mtx_unlock(&isrc_table_lock); return (0); #else return (EOPNOTSUPP); #endif } /* * Create interrupt event for interrupt source. */ static int isrc_event_create(struct intr_irqsrc *isrc) { struct intr_event *ie; int error; error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, intr_isrc_assign_cpu, "%s:", isrc->isrc_name); if (error) return (error); mtx_lock(&isrc_table_lock); /* * Make sure that we do not mix the two ways * how we handle interrupt sources. Let contested event wins. */ #ifdef INTR_SOLO if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { #else if (isrc->isrc_event != NULL) { #endif mtx_unlock(&isrc_table_lock); intr_event_destroy(ie); return (isrc->isrc_event != NULL ? EBUSY : 0); } isrc->isrc_event = ie; mtx_unlock(&isrc_table_lock); return (0); } #ifdef notyet /* * Destroy interrupt event for interrupt source. */ static void isrc_event_destroy(struct intr_irqsrc *isrc) { struct intr_event *ie; mtx_lock(&isrc_table_lock); ie = isrc->isrc_event; isrc->isrc_event = NULL; mtx_unlock(&isrc_table_lock); if (ie != NULL) intr_event_destroy(ie); } #endif /* * Add handler to interrupt source. */ static int isrc_add_handler(struct intr_irqsrc *isrc, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) { int error; if (isrc->isrc_event == NULL) { error = isrc_event_create(isrc); if (error) return (error); } error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, arg, intr_priority(flags), flags, cookiep); if (error == 0) { mtx_lock(&isrc_table_lock); intrcnt_updatename(isrc); mtx_unlock(&isrc_table_lock); } return (error); } /* * Lookup interrupt controller locked. */ static inline struct intr_pic * pic_lookup_locked(device_t dev, intptr_t xref, int flags) { struct intr_pic *pic; mtx_assert(&pic_list_lock, MA_OWNED); if (dev == NULL && xref == 0) return (NULL); /* Note that pic->pic_dev is never NULL on registered PIC. */ SLIST_FOREACH(pic, &pic_list, pic_next) { if ((pic->pic_flags & FLAG_TYPE_MASK) != (flags & FLAG_TYPE_MASK)) continue; if (dev == NULL) { if (xref == pic->pic_xref) return (pic); } else if (xref == 0 || pic->pic_xref == 0) { if (dev == pic->pic_dev) return (pic); } else if (xref == pic->pic_xref && dev == pic->pic_dev) return (pic); } return (NULL); } /* * Lookup interrupt controller. */ static struct intr_pic * pic_lookup(device_t dev, intptr_t xref, int flags) { struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref, flags); mtx_unlock(&pic_list_lock); return (pic); } /* * Create interrupt controller. */ static struct intr_pic * pic_create(device_t dev, intptr_t xref, int flags) { struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref, flags); if (pic != NULL) { mtx_unlock(&pic_list_lock); return (pic); } pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); if (pic == NULL) { mtx_unlock(&pic_list_lock); return (NULL); } pic->pic_xref = xref; pic->pic_dev = dev; pic->pic_flags = flags; mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); SLIST_INSERT_HEAD(&pic_list, pic, pic_next); mtx_unlock(&pic_list_lock); return (pic); } #ifdef notyet /* * Destroy interrupt controller. */ static void pic_destroy(device_t dev, intptr_t xref, int flags) { struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref, flags); if (pic == NULL) { mtx_unlock(&pic_list_lock); return; } SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); mtx_unlock(&pic_list_lock); free(pic, M_INTRNG); } #endif /* * Register interrupt controller. */ struct intr_pic * intr_pic_register(device_t dev, intptr_t xref) { struct intr_pic *pic; if (dev == NULL) return (NULL); pic = pic_create(dev, xref, FLAG_PIC); if (pic == NULL) return (NULL); debugf("PIC %p registered for %s \n", pic, device_get_nameunit(dev), dev, (uintmax_t)xref); return (pic); } /* * Unregister interrupt controller. */ int intr_pic_deregister(device_t dev, intptr_t xref) { panic("%s: not implemented", __func__); } /* * Mark interrupt controller (itself) as a root one. * * Note that only an interrupt controller can really know its position * in interrupt controller's tree. So root PIC must claim itself as a root. * * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, * page 30: * "The root of the interrupt tree is determined when traversal * of the interrupt tree reaches an interrupt controller node without * an interrupts property and thus no explicit interrupt parent." */ int intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, - void *arg, u_int ipicount) + void *arg) { struct intr_pic *pic; pic = pic_lookup(dev, xref, FLAG_PIC); if (pic == NULL) { device_printf(dev, "not registered\n"); return (EINVAL); } KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, ("%s: Found a non-PIC controller: %s", __func__, device_get_name(pic->pic_dev))); if (filter == NULL) { device_printf(dev, "filter missing\n"); return (EINVAL); } /* * Only one interrupt controllers could be on the root for now. * Note that we further suppose that there is not threaded interrupt * routine (handler) on the root. See intr_irq_handler(). */ if (intr_irq_root_dev != NULL) { device_printf(dev, "another root already set\n"); return (EBUSY); } intr_irq_root_dev = dev; irq_root_filter = filter; irq_root_arg = arg; - irq_root_ipicount = ipicount; debugf("irq root set to %s\n", device_get_nameunit(dev)); return (0); } /* * Add a handler to manage a sub range of a parents interrupts. */ int intr_pic_add_handler(device_t parent, struct intr_pic *pic, intr_child_irq_filter_t *filter, void *arg, uintptr_t start, uintptr_t length) { struct intr_pic *parent_pic; struct intr_pic_child *newchild; #ifdef INVARIANTS struct intr_pic_child *child; #endif /* Find the parent PIC */ parent_pic = pic_lookup(parent, 0, FLAG_PIC); if (parent_pic == NULL) return (ENXIO); newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO); newchild->pc_pic = pic; newchild->pc_filter = filter; newchild->pc_filter_arg = arg; newchild->pc_start = start; newchild->pc_length = length; mtx_lock_spin(&parent_pic->pic_child_lock); #ifdef INVARIANTS SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", __func__)); } #endif SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); mtx_unlock_spin(&parent_pic->pic_child_lock); return (0); } static int intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, struct intr_irqsrc **isrc) { struct intr_pic *pic; struct intr_map_data_msi *msi; if (data == NULL) return (EINVAL); pic = pic_lookup(dev, xref, (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC); if (pic == NULL) return (ESRCH); switch (data->type) { case INTR_MAP_DATA_MSI: KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); msi = (struct intr_map_data_msi *)data; *isrc = msi->isrc; return (0); default: KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, ("%s: Found a non-PIC controller: %s", __func__, device_get_name(pic->pic_dev))); return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); } } bool intr_is_per_cpu(struct resource *res) { u_int res_id; struct intr_irqsrc *isrc; res_id = (u_int)rman_get_start(res); isrc = intr_map_get_isrc(res_id); if (isrc == NULL) panic("Attempt to get isrc for non-active resource id: %u\n", res_id); return ((isrc->isrc_flags & INTR_ISRCF_PPI) != 0); } int intr_activate_irq(device_t dev, struct resource *res) { device_t map_dev; intptr_t map_xref; struct intr_map_data *data; struct intr_irqsrc *isrc; u_int res_id; int error; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); res_id = (u_int)rman_get_start(res); if (intr_map_get_isrc(res_id) != NULL) panic("Attempt to double activation of resource id: %u\n", res_id); intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); error = intr_resolve_irq(map_dev, map_xref, data, &isrc); if (error != 0) { free(data, M_INTRNG); /* XXX TODO DISCONECTED PICs */ /* if (error == EINVAL) return(0); */ return (error); } intr_map_set_isrc(res_id, isrc); rman_set_virtual(res, data); return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); } int intr_deactivate_irq(device_t dev, struct resource *res) { struct intr_map_data *data; struct intr_irqsrc *isrc; u_int res_id; int error; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); res_id = (u_int)rman_get_start(res); isrc = intr_map_get_isrc(res_id); if (isrc == NULL) panic("Attempt to deactivate non-active resource id: %u\n", res_id); data = rman_get_virtual(res); error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); intr_map_set_isrc(res_id, NULL); rman_set_virtual(res, NULL); free(data, M_INTRNG); return (error); } int intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, driver_intr_t hand, void *arg, int flags, void **cookiep) { int error; struct intr_map_data *data; struct intr_irqsrc *isrc; const char *name; u_int res_id; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); res_id = (u_int)rman_get_start(res); isrc = intr_map_get_isrc(res_id); if (isrc == NULL) { /* XXX TODO DISCONECTED PICs */ return (EINVAL); } data = rman_get_virtual(res); name = device_get_nameunit(dev); #ifdef INTR_SOLO /* * Standard handling is done through MI interrupt framework. However, * some interrupts could request solely own special handling. This * non standard handling can be used for interrupt controllers without * handler (filter only), so in case that interrupt controllers are * chained, MI interrupt framework is called only in leaf controller. * * Note that root interrupt controller routine is served as well, * however in intr_irq_handler(), i.e. main system dispatch routine. */ if (flags & INTR_SOLO && hand != NULL) { debugf("irq %u cannot solo on %s\n", irq, name); return (EINVAL); } if (flags & INTR_SOLO) { error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, arg, cookiep); debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error, name); } else #endif { error = isrc_add_handler(isrc, name, filt, hand, arg, flags, cookiep); debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name); } if (error != 0) return (error); mtx_lock(&isrc_table_lock); error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); if (error == 0) { isrc->isrc_handlers++; if (isrc->isrc_handlers == 1) PIC_ENABLE_INTR(isrc->isrc_dev, isrc); } mtx_unlock(&isrc_table_lock); if (error != 0) intr_event_remove_handler(*cookiep); return (error); } int intr_teardown_irq(device_t dev, struct resource *res, void *cookie) { int error; struct intr_map_data *data; struct intr_irqsrc *isrc; u_int res_id; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); res_id = (u_int)rman_get_start(res); isrc = intr_map_get_isrc(res_id); if (isrc == NULL || isrc->isrc_handlers == 0) return (EINVAL); data = rman_get_virtual(res); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) { if (isrc != cookie) return (EINVAL); mtx_lock(&isrc_table_lock); isrc->isrc_filter = NULL; isrc->isrc_arg = NULL; isrc->isrc_handlers = 0; PIC_DISABLE_INTR(isrc->isrc_dev, isrc); PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); isrc_update_name(isrc, NULL); mtx_unlock(&isrc_table_lock); return (0); } #endif if (isrc != intr_handler_source(cookie)) return (EINVAL); error = intr_event_remove_handler(cookie); if (error == 0) { mtx_lock(&isrc_table_lock); isrc->isrc_handlers--; if (isrc->isrc_handlers == 0) PIC_DISABLE_INTR(isrc->isrc_dev, isrc); PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); intrcnt_updatename(isrc); mtx_unlock(&isrc_table_lock); } return (error); } int intr_describe_irq(device_t dev, struct resource *res, void *cookie, const char *descr) { int error; struct intr_irqsrc *isrc; u_int res_id; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); res_id = (u_int)rman_get_start(res); isrc = intr_map_get_isrc(res_id); if (isrc == NULL || isrc->isrc_handlers == 0) return (EINVAL); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) { if (isrc != cookie) return (EINVAL); mtx_lock(&isrc_table_lock); isrc_update_name(isrc, descr); mtx_unlock(&isrc_table_lock); return (0); } #endif error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); if (error == 0) { mtx_lock(&isrc_table_lock); intrcnt_updatename(isrc); mtx_unlock(&isrc_table_lock); } return (error); } #ifdef SMP int intr_bind_irq(device_t dev, struct resource *res, int cpu) { struct intr_irqsrc *isrc; u_int res_id; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); res_id = (u_int)rman_get_start(res); isrc = intr_map_get_isrc(res_id); if (isrc == NULL || isrc->isrc_handlers == 0) return (EINVAL); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) return (intr_isrc_assign_cpu(isrc, cpu)); #endif return (intr_event_bind(isrc->isrc_event, cpu)); } /* * Return the CPU that the next interrupt source should use. * For now just returns the next CPU according to round-robin. */ u_int intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) { u_int cpu; KASSERT(!CPU_EMPTY(cpumask), ("%s: Empty CPU mask", __func__)); if (!irq_assign_cpu || mp_ncpus == 1) { cpu = PCPU_GET(cpuid); if (CPU_ISSET(cpu, cpumask)) return (curcpu); return (CPU_FFS(cpumask) - 1); } do { last_cpu++; if (last_cpu > mp_maxid) last_cpu = 0; } while (!CPU_ISSET(last_cpu, cpumask)); return (last_cpu); } #ifndef EARLY_AP_STARTUP /* * Distribute all the interrupt sources among the available * CPUs once the AP's have been launched. */ static void intr_irq_shuffle(void *arg __unused) { struct intr_irqsrc *isrc; u_int i; if (mp_ncpus == 1) return; mtx_lock(&isrc_table_lock); irq_assign_cpu = true; for (i = 0; i < intr_nirq; i++) { isrc = irq_sources[i]; if (isrc == NULL || isrc->isrc_handlers == 0 || isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) continue; if (isrc->isrc_event != NULL && isrc->isrc_flags & INTR_ISRCF_BOUND && isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) panic("%s: CPU inconsistency", __func__); if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) CPU_ZERO(&isrc->isrc_cpu); /* start again */ /* * We are in wicked position here if the following call fails * for bound ISRC. The best thing we can do is to clear * isrc_cpu so inconsistency with ie_cpu will be detectable. */ if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) CPU_ZERO(&isrc->isrc_cpu); } mtx_unlock(&isrc_table_lock); } SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); #endif /* !EARLY_AP_STARTUP */ #else u_int intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) { return (PCPU_GET(cpuid)); } #endif /* SMP */ /* * Allocate memory for new intr_map_data structure. * Initialize common fields. */ struct intr_map_data * intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) { struct intr_map_data *data; data = malloc(len, M_INTRNG, flags); data->type = type; data->len = len; return (data); } void intr_free_intr_map_data(struct intr_map_data *data) { free(data, M_INTRNG); } /* * Register a MSI/MSI-X interrupt controller */ int intr_msi_register(device_t dev, intptr_t xref) { struct intr_pic *pic; if (dev == NULL) return (EINVAL); pic = pic_create(dev, xref, FLAG_MSI); if (pic == NULL) return (ENOMEM); debugf("PIC %p registered for %s \n", pic, device_get_nameunit(dev), dev, (uintmax_t)xref); return (0); } int intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, int maxcount, int *irqs) { struct iommu_domain *domain; struct intr_irqsrc **isrc; struct intr_pic *pic; device_t pdev; struct intr_map_data_msi *msi; int err, i; pic = pic_lookup(NULL, xref, FLAG_MSI); if (pic == NULL) return (ESRCH); KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); /* * If this is the first time we have used this context ask the * interrupt controller to map memory the msi source will need. */ err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); if (err != 0) return (err); isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); if (err != 0) { free(isrc, M_INTRNG); return (err); } for (i = 0; i < count; i++) { isrc[i]->isrc_iommu = domain; msi = (struct intr_map_data_msi *)intr_alloc_map_data( INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); msi-> isrc = isrc[i]; irqs[i] = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); } free(isrc, M_INTRNG); return (err); } int intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, int *irqs) { struct intr_irqsrc **isrc; struct intr_pic *pic; struct intr_map_data_msi *msi; int i, err; pic = pic_lookup(NULL, xref, FLAG_MSI); if (pic == NULL) return (ESRCH); KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); for (i = 0; i < count; i++) { msi = (struct intr_map_data_msi *) intr_map_get_map_data(irqs[i]); KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, ("%s: irq %d map data is not MSI", __func__, irqs[i])); isrc[i] = msi->isrc; } MSI_IOMMU_DEINIT(pic->pic_dev, child); err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); for (i = 0; i < count; i++) { if (isrc[i] != NULL) intr_unmap_irq(irqs[i]); } free(isrc, M_INTRNG); return (err); } int intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) { struct iommu_domain *domain; struct intr_irqsrc *isrc; struct intr_pic *pic; device_t pdev; struct intr_map_data_msi *msi; int err; pic = pic_lookup(NULL, xref, FLAG_MSI); if (pic == NULL) return (ESRCH); KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); /* * If this is the first time we have used this context ask the * interrupt controller to map memory the msi source will need. */ err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); if (err != 0) return (err); err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); if (err != 0) return (err); isrc->isrc_iommu = domain; msi = (struct intr_map_data_msi *)intr_alloc_map_data( INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); msi->isrc = isrc; *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); return (0); } int intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) { struct intr_irqsrc *isrc; struct intr_pic *pic; struct intr_map_data_msi *msi; int err; pic = pic_lookup(NULL, xref, FLAG_MSI); if (pic == NULL) return (ESRCH); KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); msi = (struct intr_map_data_msi *) intr_map_get_map_data(irq); KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, ("%s: irq %d map data is not MSI", __func__, irq)); isrc = msi->isrc; if (isrc == NULL) { intr_unmap_irq(irq); return (EINVAL); } MSI_IOMMU_DEINIT(pic->pic_dev, child); err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); intr_unmap_irq(irq); return (err); } int intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, uint64_t *addr, uint32_t *data) { struct intr_irqsrc *isrc; struct intr_pic *pic; int err; pic = pic_lookup(NULL, xref, FLAG_MSI); if (pic == NULL) return (ESRCH); KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); isrc = intr_map_get_isrc(irq); if (isrc == NULL) return (EINVAL); err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); #ifdef IOMMU if (isrc->isrc_iommu != NULL) iommu_translate_msi(isrc->isrc_iommu, addr); #endif return (err); } void dosoftints(void); void dosoftints(void) { } #ifdef SMP /* * Init interrupt controller on another CPU. */ void intr_pic_init_secondary(void) { /* * QQQ: Only root PIC is aware of other CPUs ??? */ KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); //mtx_lock(&isrc_table_lock); PIC_INIT_SECONDARY(intr_irq_root_dev); //mtx_unlock(&isrc_table_lock); } #endif #ifdef DDB DB_SHOW_COMMAND_FLAGS(irqs, db_show_irqs, DB_CMD_MEMSAFE) { u_int i, irqsum; u_long num; struct intr_irqsrc *isrc; for (irqsum = 0, i = 0; i < intr_nirq; i++) { isrc = irq_sources[i]; if (isrc == NULL) continue; num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, isrc->isrc_name, isrc->isrc_cpu.__bits[0], isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); irqsum += num; } db_printf("irq total %u\n", irqsum); } #endif /* * Interrupt mapping table functions. * * Please, keep this part separately, it can be transformed to * extension of standard resources. */ struct intr_map_entry { device_t dev; intptr_t xref; struct intr_map_data *map_data; struct intr_irqsrc *isrc; /* XXX TODO DISCONECTED PICs */ /*int flags */ }; /* XXX Convert irq_map[] to dynamicaly expandable one. */ static struct intr_map_entry **irq_map; static u_int irq_map_count; static u_int irq_map_first_free_idx; static struct mtx irq_map_lock; static struct intr_irqsrc * intr_map_get_isrc(u_int res_id) { struct intr_irqsrc *isrc; isrc = NULL; mtx_lock(&irq_map_lock); if (res_id < irq_map_count && irq_map[res_id] != NULL) isrc = irq_map[res_id]->isrc; mtx_unlock(&irq_map_lock); return (isrc); } static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) { mtx_lock(&irq_map_lock); if (res_id < irq_map_count && irq_map[res_id] != NULL) irq_map[res_id]->isrc = isrc; mtx_unlock(&irq_map_lock); } /* * Get a copy of intr_map_entry data */ static struct intr_map_data * intr_map_get_map_data(u_int res_id) { struct intr_map_data *data; data = NULL; mtx_lock(&irq_map_lock); if (res_id >= irq_map_count || irq_map[res_id] == NULL) panic("Attempt to copy invalid resource id: %u\n", res_id); data = irq_map[res_id]->map_data; mtx_unlock(&irq_map_lock); return (data); } /* * Get a copy of intr_map_entry data */ static void intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, struct intr_map_data **data) { size_t len; len = 0; mtx_lock(&irq_map_lock); if (res_id >= irq_map_count || irq_map[res_id] == NULL) panic("Attempt to copy invalid resource id: %u\n", res_id); if (irq_map[res_id]->map_data != NULL) len = irq_map[res_id]->map_data->len; mtx_unlock(&irq_map_lock); if (len == 0) *data = NULL; else *data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO); mtx_lock(&irq_map_lock); if (irq_map[res_id] == NULL) panic("Attempt to copy invalid resource id: %u\n", res_id); if (len != 0) { if (len != irq_map[res_id]->map_data->len) panic("Resource id: %u has changed.\n", res_id); memcpy(*data, irq_map[res_id]->map_data, len); } *map_dev = irq_map[res_id]->dev; *map_xref = irq_map[res_id]->xref; mtx_unlock(&irq_map_lock); } /* * Allocate and fill new entry in irq_map table. */ u_int intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) { u_int i; struct intr_map_entry *entry; /* Prepare new entry first. */ entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO); entry->dev = dev; entry->xref = xref; entry->map_data = data; entry->isrc = NULL; mtx_lock(&irq_map_lock); for (i = irq_map_first_free_idx; i < irq_map_count; i++) { if (irq_map[i] == NULL) { irq_map[i] = entry; irq_map_first_free_idx = i + 1; mtx_unlock(&irq_map_lock); return (i); } } for (i = 0; i < irq_map_first_free_idx; i++) { if (irq_map[i] == NULL) { irq_map[i] = entry; irq_map_first_free_idx = i + 1; mtx_unlock(&irq_map_lock); return (i); } } mtx_unlock(&irq_map_lock); /* XXX Expand irq_map table */ panic("IRQ mapping table is full."); } /* * Remove and free mapping entry. */ void intr_unmap_irq(u_int res_id) { struct intr_map_entry *entry; mtx_lock(&irq_map_lock); if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) panic("Attempt to unmap invalid resource id: %u\n", res_id); entry = irq_map[res_id]; irq_map[res_id] = NULL; irq_map_first_free_idx = res_id; mtx_unlock(&irq_map_lock); intr_free_intr_map_data(entry->map_data); free(entry, M_INTRNG); } /* * Clone mapping entry. */ u_int intr_map_clone_irq(u_int old_res_id) { device_t map_dev; intptr_t map_xref; struct intr_map_data *data; intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); return (intr_map_irq(map_dev, map_xref, data)); } static void intr_map_init(void *dummy __unused) { mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); irq_map_count = 2 * intr_nirq; irq_map = mallocarray(irq_map_count, sizeof(struct intr_map_entry*), M_INTRNG, M_WAITOK | M_ZERO); } SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); diff --git a/sys/riscv/riscv/plic.c b/sys/riscv/riscv/plic.c index 90f79cf9b3aa..1e8a03127038 100644 --- a/sys/riscv/riscv/plic.c +++ b/sys/riscv/riscv/plic.c @@ -1,468 +1,468 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2018 Ruslan Bukin * All rights reserved. * Copyright (c) 2019 Mitchell Horne * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory (Department of Computer Science * and Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of * the DARPA SSITH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #define PLIC_MAX_IRQS 1024 #define PLIC_PRIORITY_BASE 0x000000U #define PLIC_ENABLE_BASE 0x002000U #define PLIC_ENABLE_STRIDE 0x80U #define PLIC_CONTEXT_BASE 0x200000U #define PLIC_CONTEXT_STRIDE 0x1000U #define PLIC_CONTEXT_THRESHOLD 0x0U #define PLIC_CONTEXT_CLAIM 0x4U #define PLIC_PRIORITY(n) (PLIC_PRIORITY_BASE + (n) * sizeof(uint32_t)) #define PLIC_ENABLE(sc, n, h) \ (sc->contexts[h].enable_offset + ((n) / 32) * sizeof(uint32_t)) #define PLIC_THRESHOLD(sc, h) \ (sc->contexts[h].context_offset + PLIC_CONTEXT_THRESHOLD) #define PLIC_CLAIM(sc, h) \ (sc->contexts[h].context_offset + PLIC_CONTEXT_CLAIM) static pic_disable_intr_t plic_disable_intr; static pic_enable_intr_t plic_enable_intr; static pic_map_intr_t plic_map_intr; static pic_setup_intr_t plic_setup_intr; static pic_post_ithread_t plic_post_ithread; static pic_pre_ithread_t plic_pre_ithread; static pic_bind_intr_t plic_bind_intr; struct plic_irqsrc { struct intr_irqsrc isrc; u_int irq; }; struct plic_context { bus_size_t enable_offset; bus_size_t context_offset; }; struct plic_softc { device_t dev; struct resource * intc_res; struct plic_irqsrc isrcs[PLIC_MAX_IRQS]; struct plic_context contexts[MAXCPU]; int ndev; }; #define RD4(sc, reg) \ bus_read_4(sc->intc_res, (reg)) #define WR4(sc, reg, val) \ bus_write_4(sc->intc_res, (reg), (val)) static u_int plic_irq_cpu; static int riscv_hartid_to_cpu(int hartid) { int i; CPU_FOREACH(i) { if (pcpu_find(i)->pc_hart == hartid) return (i); } return (-1); } static int plic_get_hartid(device_t dev, phandle_t intc) { int hart; /* Check the interrupt controller layout. */ if (OF_searchencprop(intc, "#interrupt-cells", &hart, sizeof(hart)) == -1) { device_printf(dev, "Could not find #interrupt-cells for phandle %u\n", intc); return (-1); } /* * The parent of the interrupt-controller is the CPU we are * interested in, so search for its hart ID. */ if (OF_searchencprop(OF_parent(intc), "reg", (pcell_t *)&hart, sizeof(hart)) == -1) { device_printf(dev, "Could not find hartid\n"); return (-1); } return (hart); } static inline void plic_irq_dispatch(struct plic_softc *sc, u_int irq, struct trapframe *tf) { struct plic_irqsrc *src; src = &sc->isrcs[irq]; if (intr_isrc_dispatch(&src->isrc, tf) != 0) device_printf(sc->dev, "Stray irq %u detected\n", irq); } static int plic_intr(void *arg) { struct plic_softc *sc; struct trapframe *tf; uint32_t pending; uint32_t cpu; sc = arg; cpu = PCPU_GET(cpuid); /* Claim any pending interrupt. */ pending = RD4(sc, PLIC_CLAIM(sc, cpu)); if (pending) { tf = curthread->td_intr_frame; plic_irq_dispatch(sc, pending, tf); } return (FILTER_HANDLED); } static void plic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; WR4(sc, PLIC_PRIORITY(src->irq), 0); } static void plic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; WR4(sc, PLIC_PRIORITY(src->irq), 1); } static int plic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct intr_map_data_fdt *daf; struct plic_softc *sc; sc = device_get_softc(dev); if (data->type != INTR_MAP_DATA_FDT) return (ENOTSUP); daf = (struct intr_map_data_fdt *)data; if (daf->ncells != 1 || daf->cells[0] > sc->ndev) return (EINVAL); *isrcp = &sc->isrcs[daf->cells[0]].isrc; return (0); } static int plic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "riscv,plic0") && !ofw_bus_is_compatible(dev, "sifive,plic-1.0.0")) return (ENXIO); device_set_desc(dev, "RISC-V PLIC"); return (BUS_PROBE_DEFAULT); } static int plic_attach(device_t dev) { struct plic_irqsrc *isrcs; struct plic_softc *sc; struct intr_pic *pic; pcell_t *cells; uint32_t irq; const char *name; phandle_t node; phandle_t xref; uint32_t cpu; int error; int rid; int nintr; int context; int i; int hart; sc = device_get_softc(dev); sc->dev = dev; node = ofw_bus_get_node(dev); if ((OF_getencprop(node, "riscv,ndev", &sc->ndev, sizeof(sc->ndev))) < 0) { device_printf(dev, "Error: could not get number of devices\n"); return (ENXIO); } if (sc->ndev >= PLIC_MAX_IRQS) { device_printf(dev, "Error: invalid ndev (%d)\n", sc->ndev); return (ENXIO); } /* Request memory resources */ rid = 0; sc->intc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->intc_res == NULL) { device_printf(dev, "Error: could not allocate memory resources\n"); return (ENXIO); } /* Register the interrupt sources */ isrcs = sc->isrcs; name = device_get_nameunit(sc->dev); for (irq = 1; irq <= sc->ndev; irq++) { isrcs[irq].irq = irq; error = intr_isrc_register(&isrcs[irq].isrc, sc->dev, 0, "%s,%u", name, irq); if (error != 0) return (error); WR4(sc, PLIC_PRIORITY(irq), 0); } /* * Calculate the per-cpu enable and context register offsets. * * This is tricky for a few reasons. The PLIC divides the interrupt * enable, threshold, and claim bits by "context", where each context * routes to a Core-Local Interrupt Controller (CLIC). * * The tricky part is that the PLIC spec imposes no restrictions on how * these contexts are laid out. So for example, there is no guarantee * that each CPU will have both a machine mode and supervisor context, * or that different PLIC implementations will organize the context * registers in the same way. On top of this, we must handle the fact * that cpuid != hartid, as they may have been renumbered during boot. * We perform the following steps: * * 1. Examine the PLIC's "interrupts-extended" property and skip any * entries that are not for supervisor external interrupts. * * 2. Walk up the device tree to find the corresponding CPU, and grab * it's hart ID. * * 3. Convert the hart to a cpuid, and calculate the register offsets * based on the context number. */ nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", sizeof(uint32_t), (void **)&cells); if (nintr <= 0) { device_printf(dev, "Could not read interrupts-extended\n"); return (ENXIO); } /* interrupts-extended is a list of phandles and interrupt types. */ for (i = 0, context = 0; i < nintr; i += 2, context++) { /* Skip M-mode external interrupts */ if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR) continue; /* Get the hart ID from the CLIC's phandle. */ hart = plic_get_hartid(dev, OF_node_from_xref(cells[i])); if (hart < 0) { OF_prop_free(cells); return (ENXIO); } /* Get the corresponding cpuid. */ cpu = riscv_hartid_to_cpu(hart); if (cpu < 0) { device_printf(dev, "Invalid hart!\n"); OF_prop_free(cells); return (ENXIO); } /* Set the enable and context register offsets for the CPU. */ sc->contexts[cpu].enable_offset = PLIC_ENABLE_BASE + context * PLIC_ENABLE_STRIDE; sc->contexts[cpu].context_offset = PLIC_CONTEXT_BASE + context * PLIC_CONTEXT_STRIDE; } OF_prop_free(cells); /* Set the threshold for each CPU to accept all priorities. */ CPU_FOREACH(cpu) WR4(sc, PLIC_THRESHOLD(sc, cpu), 0); xref = OF_xref_from_node(node); pic = intr_pic_register(sc->dev, xref); if (pic == NULL) return (ENXIO); csr_set(sie, SIE_SEIE); - return (intr_pic_claim_root(sc->dev, xref, plic_intr, sc, 0)); + return (intr_pic_claim_root(sc->dev, xref, plic_intr, sc)); } static void plic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { plic_disable_intr(dev, isrc); } static void plic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; uint32_t cpu; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; cpu = CPU_FFS(&isrc->isrc_cpu) - 1; /* Complete the interrupt. */ WR4(sc, PLIC_CLAIM(sc, cpu), src->irq); plic_enable_intr(dev, isrc); } static int plic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { CPU_ZERO(&isrc->isrc_cpu); plic_bind_intr(dev, isrc); return (0); } static int plic_bind_intr(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; uint32_t reg; u_int cpu; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; /* Disable the interrupt source on all CPUs. */ CPU_FOREACH(cpu) { reg = RD4(sc, PLIC_ENABLE(sc, src->irq, cpu)); reg &= ~(1 << (src->irq % 32)); WR4(sc, PLIC_ENABLE(sc, src->irq, cpu), reg); } if (CPU_EMPTY(&isrc->isrc_cpu)) { cpu = plic_irq_cpu = intr_irq_next_cpu(plic_irq_cpu, &all_cpus); CPU_SETOF(cpu, &isrc->isrc_cpu); } else { /* * We will only bind to a single CPU so select the first * CPU found. */ cpu = CPU_FFS(&isrc->isrc_cpu) - 1; } /* Enable the interrupt on the selected CPU only. */ reg = RD4(sc, PLIC_ENABLE(sc, src->irq, cpu)); reg |= (1 << (src->irq % 32)); WR4(sc, PLIC_ENABLE(sc, src->irq, cpu), reg); return (0); } static device_method_t plic_methods[] = { DEVMETHOD(device_probe, plic_probe), DEVMETHOD(device_attach, plic_attach), DEVMETHOD(pic_disable_intr, plic_disable_intr), DEVMETHOD(pic_enable_intr, plic_enable_intr), DEVMETHOD(pic_map_intr, plic_map_intr), DEVMETHOD(pic_pre_ithread, plic_pre_ithread), DEVMETHOD(pic_post_ithread, plic_post_ithread), DEVMETHOD(pic_post_filter, plic_post_ithread), DEVMETHOD(pic_setup_intr, plic_setup_intr), DEVMETHOD(pic_bind_intr, plic_bind_intr), DEVMETHOD_END }; static driver_t plic_driver = { "plic", plic_methods, sizeof(struct plic_softc), }; EARLY_DRIVER_MODULE(plic, simplebus, plic_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/sys/intr.h b/sys/sys/intr.h index e91a661673d0..bdc693e6cb60 100644 --- a/sys/sys/intr.h +++ b/sys/sys/intr.h @@ -1,168 +1,168 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015-2016 Svatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_INTR_H_ #define _SYS_INTR_H_ #ifndef INTRNG #error Need INTRNG for this file #endif #include #define INTR_IRQ_INVALID 0xFFFFFFFF enum intr_map_data_type { INTR_MAP_DATA_ACPI = 0, INTR_MAP_DATA_FDT, INTR_MAP_DATA_GPIO, INTR_MAP_DATA_MSI, /* Placeholders for platform specific types */ INTR_MAP_DATA_PLAT_1 = 1000, INTR_MAP_DATA_PLAT_2, INTR_MAP_DATA_PLAT_3, INTR_MAP_DATA_PLAT_4, INTR_MAP_DATA_PLAT_5, }; struct intr_map_data { size_t len; enum intr_map_data_type type; }; struct intr_map_data_msi { struct intr_map_data hdr; struct intr_irqsrc *isrc; }; #ifdef notyet #define INTR_SOLO INTR_MD1 typedef int intr_irq_filter_t(void *arg, struct trapframe *tf); #else typedef int intr_irq_filter_t(void *arg); #endif typedef int intr_child_irq_filter_t(void *arg, uintptr_t irq); #define INTR_ISRC_NAMELEN (MAXCOMLEN + 1) #define INTR_ISRCF_IPI 0x01 /* IPI interrupt */ #define INTR_ISRCF_PPI 0x02 /* PPI interrupt */ #define INTR_ISRCF_BOUND 0x04 /* bound to a CPU */ struct intr_pic; /* Interrupt source definition. */ struct intr_irqsrc { device_t isrc_dev; /* where isrc is mapped */ u_int isrc_irq; /* unique identificator */ u_int isrc_flags; char isrc_name[INTR_ISRC_NAMELEN]; cpuset_t isrc_cpu; /* on which CPUs is enabled */ u_int isrc_index; u_long * isrc_count; u_int isrc_handlers; struct intr_event * isrc_event; #ifdef INTR_SOLO intr_irq_filter_t * isrc_filter; void * isrc_arg; #endif /* Used by MSI interrupts to store the iommu details */ void * isrc_iommu; }; /* Intr interface for PIC. */ int intr_isrc_deregister(struct intr_irqsrc *); int intr_isrc_register(struct intr_irqsrc *, device_t, u_int, const char *, ...) __printflike(4, 5); #ifdef SMP bool intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu); #endif int intr_isrc_dispatch(struct intr_irqsrc *, struct trapframe *); u_int intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask); struct intr_pic *intr_pic_register(device_t, intptr_t); int intr_pic_deregister(device_t, intptr_t); -int intr_pic_claim_root(device_t, intptr_t, intr_irq_filter_t *, void *, u_int); +int intr_pic_claim_root(device_t, intptr_t, intr_irq_filter_t *, void *); int intr_pic_add_handler(device_t, struct intr_pic *, intr_child_irq_filter_t *, void *, uintptr_t, uintptr_t); bool intr_is_per_cpu(struct resource *); extern device_t intr_irq_root_dev; /* Intr interface for BUS. */ int intr_activate_irq(device_t, struct resource *); int intr_deactivate_irq(device_t, struct resource *); int intr_setup_irq(device_t, struct resource *, driver_filter_t, driver_intr_t, void *, int, void **); int intr_teardown_irq(device_t, struct resource *, void *); int intr_describe_irq(device_t, struct resource *, void *, const char *); int intr_child_irq_handler(struct intr_pic *, uintptr_t); /* Intr resources mapping. */ struct intr_map_data *intr_alloc_map_data(enum intr_map_data_type, size_t, int); void intr_free_intr_map_data(struct intr_map_data *); u_int intr_map_irq(device_t, intptr_t, struct intr_map_data *); void intr_unmap_irq(u_int ); u_int intr_map_clone_irq(u_int ); /* MSI/MSI-X handling */ int intr_msi_register(device_t, intptr_t); int intr_alloc_msi(device_t, device_t, intptr_t, int, int, int *); int intr_release_msi(device_t, device_t, intptr_t, int, int *); int intr_map_msi(device_t, device_t, intptr_t, int, uint64_t *, uint32_t *); int intr_alloc_msix(device_t, device_t, intptr_t, int *); int intr_release_msix(device_t, device_t, intptr_t, int); #ifdef SMP int intr_bind_irq(device_t, struct resource *, int); void intr_pic_init_secondary(void); /* Virtualization for interrupt source IPI counter increment. */ static inline void intr_ipi_increment_count(u_long *counter, u_int cpu) { KASSERT(cpu < MAXCPU, ("%s: too big cpu %u", __func__, cpu)); counter[cpu]++; } /* Virtualization for interrupt source IPI counters setup. */ u_long * intr_ipi_setup_counters(const char *name); #endif extern u_int intr_nirq; /* number of IRQs on intrng platforms */ #endif /* _SYS_INTR_H */