diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c --- a/sys/arm64/arm64/gic_v3.c +++ b/sys/arm64/arm64/gic_v3.c @@ -494,6 +494,9 @@ case GICV3_IVAR_REDIST: *result = (uintptr_t)&sc->gic_redists.pcpu[PCPU_GET(cpuid)]; return (0); + case GICV3_IVAR_FLAGS: + *result = sc->gic_flags; + return (0); case GIC_IVAR_SUPPORT_LPIS: *result = (gic_d_read(sc, 4, GICD_TYPER) & GICD_TYPER_LPIS) != 0; @@ -530,6 +533,7 @@ switch(which) { case GICV3_IVAR_NIRQS: case GICV3_IVAR_REDIST: + case GICV3_IVAR_FLAGS: case GIC_IVAR_HW_REV: case GIC_IVAR_BUS: return (EINVAL); diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c --- a/sys/arm64/arm64/gic_v3_fdt.c +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -110,7 +110,7 @@ { struct gic_v3_softc *sc; pcell_t redist_regions; - intptr_t xref; + phandle_t xref, node; int err; uint32_t *mbi_ranges; ssize_t ret; @@ -118,6 +118,19 @@ sc = device_get_softc(dev); sc->dev = dev; sc->gic_bus = GIC_BUS_FDT; + node = ofw_bus_get_node(dev); + + /* + * Limit DMA shareability. "dma-noncoherent" was introduced in DT 6.15. + * For compatibility with previous versions, also use a match based on + * affected SoCs. + */ + if (OF_hasprop(node, "dma-noncoherent") || + ofw_bus_is_machine_compatible("rockchip,rk3566") || + ofw_bus_is_machine_compatible("rockchip,rk3568") || + ofw_bus_is_machine_compatible("rockchip,rk3588") || + ofw_bus_is_machine_compatible("rockchip,rk3588s")) + sc->gic_flags |= GIC_V3_FLAGS_FORCE_NOSHAREABLE; /* * Recover number of the Re-Distributor regions. diff --git a/sys/arm64/arm64/gic_v3_var.h b/sys/arm64/arm64/gic_v3_var.h --- a/sys/arm64/arm64/gic_v3_var.h +++ b/sys/arm64/arm64/gic_v3_var.h @@ -63,6 +63,8 @@ struct redist_pcpu *pcpu; }; +#define GIC_V3_FLAGS_FORCE_NOSHAREABLE 0x0000001 + struct gic_v3_softc { device_t dev; struct resource ** gic_res; @@ -92,6 +94,8 @@ int nranges; struct arm_gic_range * ranges; + + uint32_t gic_flags; }; struct gic_v3_devinfo { @@ -108,9 +112,11 @@ #define GICV3_IVAR_NIRQS 1000 /* 1001 was GICV3_IVAR_REDIST_VADDR */ #define GICV3_IVAR_REDIST 1002 +#define GICV3_IVAR_FLAGS 1003 __BUS_ACCESSOR(gicv3, nirqs, GICV3, NIRQS, u_int); __BUS_ACCESSOR(gicv3, redist, GICV3, REDIST, void *); +__BUS_ACCESSOR(gicv3, flags, GICV3, FLAGS, uint32_t); /* Device methods */ int gic_v3_attach(device_t dev); diff --git a/sys/arm64/arm64/gicv3_its.c b/sys/arm64/arm64/gicv3_its.c --- a/sys/arm64/arm64/gicv3_its.c +++ b/sys/arm64/arm64/gicv3_its.c @@ -287,28 +287,50 @@ #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 #define ITS_FLAGS_LPI_PREALLOC 0x00000008 +#define ITS_FLAGS_FORCE_NOSHAREABLE 0x00000010 u_int sc_its_flags; bool trace_enable; vm_page_t ma; /* fake msi page */ + vm_paddr_t malloc_max_addr; /* max address for contigmalloc */ }; +typedef bool (its_quirk_detect_t)(device_t); typedef void (its_quirk_func_t)(device_t); -static its_quirk_func_t its_quirk_cavium_22375; + +static its_quirk_detect_t its_detect_cavium_22375; +static its_quirk_func_t its_quirk_cavium_22375; +#ifdef FDT +static its_quirk_detect_t its_detect_rk356x; +static its_quirk_func_t its_quirk_rk356x; +static its_quirk_detect_t its_detect_rk3588; +static its_quirk_func_t its_quirk_rk3588; +#endif static const struct { const char *desc; - uint32_t iidr; - uint32_t iidr_mask; + its_quirk_detect_t *detect; its_quirk_func_t *func; } its_quirks[] = { { /* Cavium ThunderX Pass 1.x */ .desc = "Cavium ThunderX errata: 22375, 24313", - .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, - GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), - .iidr_mask = ~GITS_IIDR_REVISION_MASK, + .detect = its_detect_cavium_22375, .func = its_quirk_cavium_22375, }, +#ifdef FDT + { + /* Rockchip RK356X implementation bugs */ + .desc = "RK356X ITS errata", + .detect = its_detect_rk356x, + .func = its_quirk_rk356x, + }, + { + /* Rockchip RK3588 implementation bugs */ + .desc = "RK3588 ITS errata", + .detect = its_detect_rk3588, + .func = its_quirk_rk3588, + }, +#endif }; #define gic_its_read_4(sc, reg) \ @@ -387,6 +409,13 @@ static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, sizeof(struct gicv3_its_softc)); +/* Limit maximum address for memory mapped tables and buffers */ +static vm_paddr_t +gicv3_its_limit_max_addr(struct gicv3_its_softc *sc, vm_paddr_t addr) +{ + return (sc->malloc_max_addr > addr ? addr: sc->malloc_max_addr); +} + static void gicv3_its_cmdq_init(struct gicv3_its_softc *sc) { @@ -395,7 +424,8 @@ /* Set up the command circular buffer */ sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS, - sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, + sc->sc_ds, M_WAITOK | M_ZERO, 0, + gicv3_its_limit_max_addr(sc, (1ul << 48) - 1), ITS_CMDQ_ALIGN, 0); sc->sc_its_cmd_next_idx = 0; @@ -404,8 +434,11 @@ /* Set the base of the command buffer */ reg = GITS_CBASER_VALID | (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | - cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | - (ITS_CMDQ_SIZE / 4096 - 1); + cmd_paddr | (ITS_CMDQ_SIZE / 4096 - 1); + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; + else + reg |= GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT; gic_its_write_8(sc, GITS_CBASER, reg); /* Read back to check for fixed value fields */ @@ -532,10 +565,17 @@ cache = 0; } else { devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); - cache = GITS_BASER_CACHE_RAWAWB; + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + cache = GITS_BASER_CACHE_NC; + else + cache = GITS_BASER_CACHE_RAWAWB; } sc->sc_devbits = devbits; - share = GITS_BASER_SHARE_IS; + + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + share = GITS_BASER_SHARE_NS; + else + share = GITS_BASER_SHARE_IS; for (i = 0; i < GITS_BASER_NUM; i++) { reg = gic_its_read_8(sc, GITS_BASER(i)); @@ -611,7 +651,8 @@ /* Allocate the table */ table = contigmalloc_domainset(npages * PAGE_SIZE, M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, - (1ul << 48) - 1, PAGE_SIZE_64K, 0); + gicv3_its_limit_max_addr(sc, (1ul << 48) - 1), + PAGE_SIZE_64K, 0); sc->sc_its_ptab[i].ptab_vaddr = table; sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size; @@ -741,7 +782,8 @@ * PROPBASER register later in its_init_cpu_lpi(). */ conf_table = contigmalloc(LPI_CONFTAB_SIZE, - M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, + M_GICV3_ITS, M_WAITOK, 0, + gicv3_its_limit_max_addr(sc, LPI_CONFTAB_MAX_ADDR), LPI_CONFTAB_ALIGN, 0); } sc->sc_conf_base = conf_table; @@ -766,7 +808,9 @@ sc->sc_pend_base[i] = contigmalloc( LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, - 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); + 0, + gicv3_its_limit_max_addr(sc, LPI_PENDTAB_MAX_ADDR), + LPI_PENDTAB_ALIGN, 0); /* Flush so the ITS can see the memory */ cpu_dcache_wb_range(sc->sc_pend_base[i], @@ -804,10 +848,12 @@ size = ilog2_long(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1; xbaser = vtophys(sc->sc_conf_base) | - (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | size; - + if (gicv3_get_flags(sc->dev) & GIC_V3_FLAGS_FORCE_NOSHAREABLE) + xbaser |= GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT; + else + xbaser |= GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT; gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); /* Check the cache attributes we set */ @@ -835,8 +881,11 @@ * Set the LPI pending table base */ xbaser = vtophys(sc->sc_pend_base[cpuid]) | - (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | - (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); + (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT); + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + xbaser |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; + else + xbaser |= GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT; gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); @@ -1017,6 +1066,7 @@ sc->sc_irq_length = gicv3_get_nirqs(dev); sc->sc_irq_base = GIC_FIRST_LPI; sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; + sc->malloc_max_addr = ~0; rid = 0; sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, @@ -1034,7 +1084,7 @@ CPU_COPY(&all_cpus, &sc->sc_cpus); iidr = gic_its_read_4(sc, GITS_IIDR); for (i = 0; i < nitems(its_quirks); i++) { - if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { + if (its_quirks[i].detect(dev)) { if (bootverbose) { device_printf(dev, "Applying %s\n", its_quirks[i].desc); @@ -1128,6 +1178,21 @@ return (ENXIO); } +static bool +its_detect_cavium_22375(device_t dev) +{ + uint32_t iidr; + struct gicv3_its_softc *sc; + + sc = device_get_softc(dev); + iidr = gic_its_read_4(sc, GITS_IIDR); + if ((iidr & ~GITS_IIDR_REVISION_MASK) == + GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, GITS_IIDR_PROD_THUNDER, + GITS_IIDR_VAR_THUNDER_1, 0)) + return (true); + return(false); +} + static void its_quirk_cavium_22375(device_t dev) { @@ -1151,6 +1216,46 @@ } } +#ifdef FDT +static bool +its_detect_rk356x(device_t dev) +{ + + if (ofw_bus_is_machine_compatible("rockchip,rk3566") || + ofw_bus_is_machine_compatible("rockchip,rk3568")) + return (true); + return(false); +} + +static void +its_quirk_rk356x(device_t dev) +{ + struct gicv3_its_softc *sc; + + sc = device_get_softc(dev); + sc->malloc_max_addr = (1ul << 32) - 1; +} + +static bool +its_detect_rk3588(device_t dev) +{ + + if (ofw_bus_is_machine_compatible("rockchip,rk3588") || + ofw_bus_is_machine_compatible("rockchip,rk3588s")) + return (true); + return(false); +} + +static void +its_quirk_rk3588(device_t dev) +{ + struct gicv3_its_softc *sc; + + sc = device_get_softc(dev); + sc->sc_its_flags |= ITS_FLAGS_FORCE_NOSHAREABLE; +} +#endif + static void gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) { @@ -1400,7 +1505,8 @@ shareable = false; l2_table = contigmalloc_domainset(ptable->ptab_l2_size, - M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, + M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, + gicv3_its_limit_max_addr(sc, (1ul << 48) - 1), ptable->ptab_page_size, 0); if (!shareable) @@ -1461,7 +1567,8 @@ itt_size = roundup2(MAX(nvecs, 2) * esize, 256); its_dev->itt = contigmalloc_domainset(itt_size, M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0, - LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0); + gicv3_its_limit_max_addr(sc, LPI_INT_TRANS_TAB_MAX_ADDR), + LPI_INT_TRANS_TAB_ALIGN, 0); if (its_dev->itt == NULL) { vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); free(its_dev, M_GICV3_ITS); @@ -2233,17 +2340,20 @@ gicv3_its_fdt_attach(device_t dev) { struct gicv3_its_softc *sc; - phandle_t xref; + phandle_t xref, node; int err; sc = device_get_softc(dev); sc->dev = dev; + node = ofw_bus_get_node(dev); err = gicv3_its_attach(dev); if (err != 0) return (err); + if (OF_hasprop(node, "dma-noncoherent")) + sc->sc_its_flags |= ITS_FLAGS_FORCE_NOSHAREABLE; /* Register this device as a interrupt controller */ - xref = OF_xref_from_node(ofw_bus_get_node(dev)); + xref = OF_xref_from_node(node); sc->sc_pic = intr_pic_register(dev, xref); err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);