diff --git a/sys/arm64/arm64/gicv5.c b/sys/arm64/arm64/gicv5.c new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gicv5.c @@ -0,0 +1,1511 @@ +/*- + * Copyright (c) 2015-2016 The FreeBSD Foundation + * Copyright (c) 2025 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "opt_acpi.h" +#include "opt_platform.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#ifdef FDT +#include +#include +#endif + +#include "pic_if.h" + +#include +#include "gicv5reg.h" +#include "gicv5var.h" +#include "gic_v3_var.h" /* For GICV3_IVAR_NIRQS */ + +#define GICV5_PPIS_PER_REG 64 +#define GICV5_PPI_COUNT 128 + +#define LPI_IPI_BASE 0 +#define LPI_IPI_LIMIT (LPI_IPI_BASE + (mp_maxid + 1) * INTR_IPI_COUNT) +#define LPI_IS_IPI(lpi) ((lpi) < LPI_ITS_BASE) +#define LPI_IPI_IDX(lpi) ((lpi) - LPI_IPI_BASE) +#define LPI_TO_IPI(lpi) (LPI_IPI_IDX(lpi) % INTR_IPI_COUNT) + +#define LPI_ITS_BASE (LPI_IPI_BASE + LPI_IPI_LIMIT) + +/* 2^12 LPIs should be enough for a linear table */ +#define GICV5_LPI_ID_BITS_MAX 12 + +/* log2(number of l2 entries) */ +#define LPI_IST_L2_ENTRIES_BITS(istsz, l2sz) (10 - (istsz) + 2 * (l2sz)) + +#define IRS_CFG_READ_4(_irs, _reg) \ + bus_read_4((_irs)->irs_cfg, (_reg)) +#define IRS_CFG_WRITE_4(_irs, _reg, _val) \ + bus_write_4((_irs)->irs_cfg, (_reg), (_val)) +#define IRS_CFG_READ_8(_irs, _reg) \ + bus_read_8((_irs)->irs_cfg, (_reg)) +#define IRS_CFG_WRITE_8(_irs, _reg, _val) \ + bus_write_8((_irs)->irs_cfg, (_reg), (_val)) + +struct gicv5_irs { + cpuset_t irs_cpus; + struct resource *irs_cfg; + uint64_t *ist_base; + + u_int irs_next_irq_cpu; + + int irs_cfg_rid; + u_int irs_spi_start; + u_int irs_spi_count; + + struct mtx irs_lock; + size_t irs_lpi_l2size; + u_int irs_lpi_l2bits; + bool irs_lpi_2l; +}; + +struct gicv5_irqsrc { + struct gicv5_base_irqsrc gi_isrc; + struct gicv5_irs *gi_irs; + enum intr_polarity gi_pol; + enum intr_trigger gi_trig; +}; + +static __read_mostly int *gicv5_iaffids; + +static bus_print_child_t gicv5_print_child; +static bus_read_ivar_t gicv5_read_ivar; +static bus_get_cpus_t gicv5_get_cpus; +static bus_get_resource_list_t gicv5_get_resource_list; +static pic_disable_intr_t gicv5_disable_intr; +static pic_enable_intr_t gicv5_enable_intr; +static pic_map_intr_t gicv5_map_intr; +static pic_setup_intr_t gicv5_setup_intr; +static pic_teardown_intr_t gicv5_teardown_intr; +static pic_post_filter_t gicv5_post_filter; +static pic_post_ithread_t gicv5_post_ithread; +static pic_pre_ithread_t gicv5_pre_ithread; +static pic_bind_intr_t gicv5_bind_intr; +#ifdef SMP +static pic_init_secondary_t gicv5_init_secondary; +static pic_ipi_send_t gicv5_ipi_send; +static pic_ipi_setup_per_cpu_t gicv5_ipi_setup_per_cpu; +#endif + +static device_method_t gicv5_methods[] = { + /* Bus interface */ + DEVMETHOD(bus_print_child, gicv5_print_child), + DEVMETHOD(bus_read_ivar, gicv5_read_ivar), + DEVMETHOD(bus_get_cpus, gicv5_get_cpus), + DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), + DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + DEVMETHOD(bus_get_resource_list, gicv5_get_resource_list), + DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), + DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), + DEVMETHOD(bus_alloc_resource, bus_generic_rl_alloc_resource), + DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource), + DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), + DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), + + /* Interrupt controller interface */ + DEVMETHOD(pic_disable_intr, gicv5_disable_intr), + DEVMETHOD(pic_enable_intr, gicv5_enable_intr), + DEVMETHOD(pic_map_intr, gicv5_map_intr), + DEVMETHOD(pic_setup_intr, gicv5_setup_intr), + DEVMETHOD(pic_teardown_intr, gicv5_teardown_intr), + DEVMETHOD(pic_post_filter, gicv5_post_filter), + DEVMETHOD(pic_post_ithread, gicv5_post_ithread), + DEVMETHOD(pic_pre_ithread, gicv5_pre_ithread), + DEVMETHOD(pic_bind_intr, gicv5_bind_intr), +#ifdef SMP + DEVMETHOD(pic_init_secondary, gicv5_init_secondary), + DEVMETHOD(pic_ipi_send, gicv5_ipi_send), + DEVMETHOD(pic_ipi_setup_per_cpu, gicv5_ipi_setup_per_cpu), +#endif + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_0(gic, gicv5_driver, gicv5_methods, sizeof(struct gicv5_softc)); + +static int +gicv5_wait_for_op(struct gicv5_irs *irs, bus_size_t reg, uint32_t mask, + uint32_t *valp) +{ + uint32_t val; + int timeout; + + /* Timeout of ~10ms */ + timeout = 10000; + do { + val = IRS_CFG_READ_4(irs, reg); + if ((val & mask) != 0) { + if (valp != NULL) + *valp = val; + return (0); + } + DELAY(1); + } while (--timeout > 0); + + return (ETIMEDOUT); +} + +static int +gicv5_wait_irs_cr0_idle(struct gicv5_irs *irs) +{ + return (gicv5_wait_for_op(irs, IRS_CR0, IRS_CR0_IDLE, NULL)); +} + +static int +gicv5_wait_irs_spi_status_idle(struct gicv5_irs *irs) +{ + uint32_t val; + int error; + + error = gicv5_wait_for_op(irs, IRS_SPI_STATUSR, IRS_SPI_STATUSR_IDLE, + &val); + if (error != 0) + return (error); + + if ((val & IRS_SPI_STATUSR_V) == 0) + return (EIO); + + return (0); +} + +static void +gicv5_irs_init_ist(struct gicv5_softc *sc, struct gicv5_irs *irs, + uint64_t cfgr) +{ + IRS_CFG_WRITE_4(irs, IRS_IST_CFGR, cfgr); + + KASSERT((vtophys(irs->ist_base) & ~IRS_IST_BASER_ADDR_MASK) == 0, + ("%s: Invalid IST base address %lx", __func__, + vtophys(irs->ist_base))); + IRS_CFG_WRITE_8(irs, IRS_IST_BASER, vtophys(irs->ist_base) | + IRS_IST_BASER_VALID); + + gicv5_wait_for_op(irs, IRS_IST_STATUSR, IRS_IST_STATUSR_IDLE, NULL); +} + +static void +gicv5_irs_alloc_ist(struct gicv5_softc *sc, struct gicv5_irs *irs, + size_t size) +{ + irs->ist_base = contigmalloc(size, M_DEVBUF, M_WAITOK | M_ZERO, 0, + IRS_IST_BASER_ADDR_LIMIT, size, 0); + if (sc->gic_coherent) + /* Ensure the IRS observed zeroed memory */ + dsb(ishst); + else + cpu_dcache_wbinv_range(irs->ist_base, size); + +} + +static void +gicv5_irs_alloc_linear(struct gicv5_softc *sc, struct gicv5_irs *irs, + uint32_t *cfgrp, u_int lpi_id_bits, u_int istsz) +{ + size_t size; + uint32_t cfgr; + u_int n; + + MPASS(istsz <= IRS_IST_CFGR_ISTSZ_16_VAL); + + /* + * This is the alignment calculation from the IRS_IST_BASER + * definition. If the size is > 64 bytes then size == align. + * for sizes < 64 bytes we can just round up the size. + */ + n = MAX(5, istsz + 1 + lpi_id_bits); + size = 1ul << (n + 1); + + gicv5_irs_alloc_ist(sc, irs, size); + + irs->irs_lpi_2l = false; + + cfgr = IRS_IST_CFGR_STRUCTURE_LINEAR; + cfgr |= istsz << IRS_IST_CFGR_ISTSZ_SHIFT; + cfgr |= lpi_id_bits << IRS_IST_CFGR_LPI_ID_BITS_SHIFT; + *cfgrp = cfgr; +} + +static void +gicv5_irs_alloc_2level(struct gicv5_softc *sc, struct gicv5_irs *irs, + uint32_t *cfgrp, u_int lpi_id_bits, u_int istsz, u_int l2sz) +{ + size_t size; + uint32_t cfgr; + u_int n; + + MPASS(istsz <= IRS_IST_CFGR_ISTSZ_16_VAL); + + /* + * This is the alignment calculation from the IRS_IST_BASER + * definition. If the size is > 64 bytes then size == align. + * for sizes < 64 bytes we can just round up the size. + */ + n = MAX(5, lpi_id_bits - LPI_IST_L2_ENTRIES_BITS(istsz, l2sz) + 2); + size = 1ul << (n + 1); + + gicv5_irs_alloc_ist(sc, irs, size); + + irs->irs_lpi_l2size = 1ul << (L1_ISTE_L2_ADDR_N(l2sz) + 1); + irs->irs_lpi_l2bits = LPI_IST_L2_ENTRIES_BITS(istsz, l2sz); + irs->irs_lpi_2l = true; + + cfgr = IRS_IST_CFGR_STRUCTURE_2LVL; + cfgr |= istsz << IRS_IST_CFGR_ISTSZ_SHIFT; + cfgr |= l2sz << IRS_IST_CFGR_L2SZ_SHIFT; + cfgr |= lpi_id_bits << IRS_IST_CFGR_LPI_ID_BITS_SHIFT; + *cfgrp = cfgr; +} + +void +gicv5_irs_alloc_lpi(device_t dev, device_t child, u_int lpi) +{ + struct gicv5_softc *sc; + struct gicv5_devinfo *di; + struct gicv5_irs *irs; + void *l2_ist; + size_t size; + u_int index; + + di = device_get_ivars(child); + irs = di->di_irs; + MPASS(irs != NULL); + + /* + * If we have a linear table then we don't need to extend it, it is + * already large enough for all LPIs we could allocate. + */ + if (!irs->irs_lpi_2l) + return; + + sc = device_get_softc(dev); + index = lpi >> irs->irs_lpi_l2bits; + size = irs->irs_lpi_l2size; + + /* Check if there the l2 pointer is valid */ + if ((irs->ist_base[index] & L1_ISTE_VALID) != 0) { + return; + } + + /* Try allocating the level 2 IST */ + l2_ist = contigmalloc(size, M_DEVBUF, M_WAITOK | M_ZERO, 0, + IRS_IST_BASER_ADDR_LIMIT, size, 0); + + mtx_lock_spin(&irs->irs_lock); + + /* Chek if we won the race */ + if ((irs->ist_base[index] & L1_ISTE_VALID) != 0) { + mtx_unlock_spin(&irs->irs_lock); + free(l2_ist, M_DEVBUF); + return; + } + + irs->ist_base[index] = vtophys(l2_ist) | L1_ISTE_VALID; + if (sc->gic_coherent) { + dsb(ishst); + } else { + cpu_dcache_wbinv_range(l2_ist, size); + cpu_dcache_wb_range(&irs->ist_base[index], + sizeof(irs->ist_base[index])); + } + + IRS_CFG_WRITE_4(irs, IRS_MAP_L2_ISTR, lpi); + + gicv5_wait_for_op(irs, IRS_IST_STATUSR, IRS_IST_STATUSR_IDLE, NULL); + + if (!sc->gic_coherent) + cpu_dcache_inv_range(&irs->ist_base[index], + sizeof(irs->ist_base[index])); + mtx_unlock_spin(&irs->irs_lock); +} + +void +gicv5_irs_init(device_t dev, u_int idx, cpuset_t *cpuset) +{ + struct gicv5_softc *sc; + + sc = device_get_softc(dev); + + MPASS(idx < sc->gic_nirs); + MPASS(sc->gic_irs != NULL); + MPASS(sc->gic_irs[idx] == NULL); + + sc->gic_irs[idx] = malloc(sizeof(*sc->gic_irs[0]), M_DEVBUF, + M_ZERO | M_WAITOK); + CPU_COPY(cpuset, &sc->gic_irs[idx]->irs_cpus); + sc->gic_irs[idx]->irs_cfg_rid = idx; +} + +static void +gicv5_irs_attach(device_t dev, struct gicv5_irs *irs, u_int idx) +{ + struct gicv5_softc *sc; + const char *name; + uint64_t icc_idr0; + uint32_t cfgr, cr1, idr2; + u_int istsz, lpi_id_bits, l2sz, spi_count, spi_end; + int error, iaffid; + bool two_levels; + + sc = device_get_softc(dev); + name = device_get_nameunit(dev); + + /* The attachment needs to set this */ + MPASS(!CPU_EMPTY(&irs->irs_cpus)); + + irs->irs_cfg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &irs->irs_cfg_rid, RF_ACTIVE); + if (irs->irs_cfg == NULL) + panic("%s: Unable to allocate memory resource", + device_get_nameunit(dev)); + + /* Set the control registers */ + if (sc->gic_coherent) { + cr1 = IRS_CR1_VPED_WA | + IRS_CR1_VPED_RA | + IRS_CR1_VMD_WA | + IRS_CR1_VMD_RA | + IRS_CR1_VPET_WA | + IRS_CR1_VPET_RA | + IRS_CR1_VMT_WA | + IRS_CR1_VMT_RA | + IRS_CR1_IST_WA | + IRS_CR1_IST_RA | + IRS_CR1_IC_WB | + IRS_CR1_OC_WB | + IRS_CR1_SH_IS; + } else { + cr1 = IRS_CR1_VPED_NO_WA | + IRS_CR1_VPED_NO_RA | + IRS_CR1_VMD_NO_WA | + IRS_CR1_VMD_NO_RA | + IRS_CR1_VPET_NO_WA | + IRS_CR1_VPET_NO_RA | + IRS_CR1_VMT_NO_WA | + IRS_CR1_VMT_NO_RA | + IRS_CR1_IST_NO_WA | + IRS_CR1_IST_NO_RA | + IRS_CR1_IC_NC | + IRS_CR1_OC_NC | + IRS_CR1_SH_NS; + } + IRS_CFG_WRITE_4(irs, IRS_CR1, cr1); + IRS_CFG_WRITE_4(irs, IRS_CR0, IRS_CR0_IRSEN); + gicv5_wait_irs_cr0_idle(irs); + + idr2 = IRS_CFG_READ_4(irs, IRS_IDR2); + + two_levels = (idr2 & IRS_IDR2_IST_LEVELS) != 0; + lpi_id_bits = IRS_IDR2_ID_BITS(idr2); + + if (!two_levels) { + /* + * Limit the size of the table as we need to entierly allocate + * it for the linear table + */ + lpi_id_bits = MIN(lpi_id_bits, GICV5_LPI_ID_BITS_MAX); + /* Ensure lpi_id_bits is at least the mnimum value */ + lpi_id_bits = MAX(lpi_id_bits, IRS_IDR2_MIN_LPI_ID_BITS(idr2)); + } + + /* TODO: Should we sanitize this? */ + icc_idr0 = READ_SPECIALREG(ICC_IDR0_EL1); + switch(icc_idr0 & ICC_IDR0_ID_BITS_MASK) { + case ICC_IDR0_ID_BITS_16: + lpi_id_bits = MIN(lpi_id_bits, 16); + break; + default: + case ICC_IDR0_ID_BITS_24: + lpi_id_bits = MIN(lpi_id_bits, 24); + break; + } + + /* The IST entries contain metadata so the size will be larger */ + if ((idr2 & IRS_IRD2_ISTMD) != 0) { + uint64_t istmd_sz; + + istmd_sz = (idr2 & IRS_IDR2_ISTMD_SZ_MASK) >> + IRS_IDR2_ISTMD_SZ_SHIFT; + if (lpi_id_bits < istmd_sz) { + istsz = IRS_IST_CFGR_ISTSZ_8_VAL; + } else { + istsz = IRS_IST_CFGR_ISTSZ_16_VAL; + } + } else { + /* The default ITS entry size is 4 bytes */ + istsz = IRS_IST_CFGR_ISTSZ_4_VAL; + } + + if (two_levels) { + if ((idr2 & IRS_IDR2_IST_L2SZ_64K) != 0) + l2sz = IRS_IST_CFGR_L2SZ_64K_VAL; + else if ((idr2 & IRS_IDR2_IST_L2SZ_16K) != 0) + l2sz = IRS_IST_CFGR_L2SZ_16K_VAL; + else + l2sz = IRS_IST_CFGR_L2SZ_4K_VAL; + } + + /* + * Use 2 level tables if able, and the size is large enough for them + * to be worth it. This is based on the calculation in the GICv5 + * spec (ARM-AES-0070) 00EAC0 section 10.2.1.14 IRS_IST_CFGR. + */ + if (two_levels && + lpi_id_bits > LPI_IST_L2_ENTRIES_BITS(istsz, l2sz)) { + gicv5_irs_alloc_2level(sc, irs, &cfgr, lpi_id_bits, istsz, + l2sz); + } else { + gicv5_irs_alloc_linear(sc, irs, &cfgr, lpi_id_bits, istsz); + } + + if (idx == 0) + sc->gic_nlpis = 1u << lpi_id_bits; + else + sc->gic_nlpis = MIN(sc->gic_nlpis, 1u << lpi_id_bits); + + spi_count = IRS_CFG_READ_4(irs, IRS_IDR5) & IRS_IDR5_SPI_RANGE; + if (sc->gic_irs_irqs == NULL) { + KASSERT(idx == 0, + ("%s: Null IRS table on no-zero index (idx = %d)", + __func__, idx)); + sc->gic_spi_count = spi_count; + sc->gic_irs_irqs = mallocarray(spi_count, + sizeof(struct gicv5_irqsrc), M_DEVBUF, M_WAITOK | M_ZERO); + } + + /* Read and check the IRS SPI details */ + irs->irs_spi_start = + IRS_CFG_READ_4(irs, IRS_IDR7) & IRS_IDR7_SPI_BASE; + + irs->irs_spi_count = + IRS_CFG_READ_4(irs, IRS_IDR6) & IRS_IDR6_SPI_IRS_RANGE; + + spi_end = irs->irs_spi_start + irs->irs_spi_count; + if (spi_end > spi_count) + panic("%s: IRS %u has SPIs past global count (%u > %u)\n", + device_get_nameunit(dev), idx, spi_end, spi_count); + + gicv5_irs_init_ist(sc, irs, cfgr); + + mtx_init(&irs->irs_lock, "GICv5 IRS lock", NULL, MTX_SPIN); + + /* + * Set a valid interrupt affinity ID, even if it's for a CPU not + * attached to this IRS. + */ + iaffid = gicv5_iaffids[curcpu]; + MPASS(iaffid >= 0); + for (u_int irq = irs->irs_spi_start; irq < spi_end; irq++) { + struct gicv5_irqsrc *gi; + uint64_t cdaff, cdpri; + + gi = &sc->gic_irs_irqs[irq]; + MPASS(gi->gi_irs == NULL); + gi->gi_isrc.gbi_space = GICv5_SPI; + gi->gi_isrc.gbi_irq = irq; + gi->gi_irs = irs; + gi->gi_pol = INTR_POLARITY_CONFORM; + gi->gi_trig = INTR_TRIGGER_CONFORM; + error = intr_isrc_register(&gi->gi_isrc.gbi_isrc, dev, + 0, "%s,s%u", name, irq); + if (error != 0) + panic("%s: Unable to register SPI irq src", + device_get_nameunit(dev)); + + /* Set the base priority */ + cdpri = GIC_CDPRI_PRORITY(GICV5_PRI_LOWEST); + cdpri |= GIC_CDPRI_TYPE_SPI; + cdpri |= GIC_CDPRI_ID(irq); + WRITE_SPECIALREG(GIC_CDPRI, cdpri); + + /* Set the affinity */ + cdaff = GIC_CDAFF_IAFFID(iaffid); + cdaff |= GIC_CDAFF_TYPE_SPI; + cdaff |= GIC_CDAFF_IRM_TARGETED; + cdaff |= GIC_CDAFF_ID(irq); + WRITE_SPECIALREG(GIC_CDAFF, cdaff); + isb(); + } +} + +void +gicv5_attach(device_t dev) +{ + struct gicv5_softc *sc; + const char *name; + int error; + + sc = device_get_softc(dev); + sc->gic_dev = dev; + name = device_get_nameunit(dev); + + MPASS(gicv5_iaffids == NULL); + gicv5_iaffids = mallocarray(mp_maxid + 1, sizeof(*gicv5_iaffids), + M_DEVBUF, M_WAITOK); + for (int i = 0; i <= mp_maxid; i++) + gicv5_iaffids[i] = -1; + gicv5_iaffids[curcpu] = + ICC_IAFFIDR_IAFFID_VAL(READ_SPECIALREG(ICC_IAFFIDR_EL1)); + + for (u_int i = 0; i < sc->gic_nirs; i++) + gicv5_irs_attach(dev, sc->gic_irs[i], i); + if (bootverbose) + device_printf(dev, "Limited to %u LPIs\n", sc->gic_nlpis); + + sc->gic_ppi_irqs = mallocarray(GICV5_PPI_COUNT, + sizeof(struct gicv5_irqsrc), M_DEVBUF, M_ZERO | M_WAITOK); + for (u_int irq = 0; irq < GICV5_PPI_COUNT; irq++) { + struct intr_irqsrc *isrc; + + sc->gic_ppi_irqs[irq].gi_irs = NULL; + sc->gic_ppi_irqs[irq].gi_isrc.gbi_space = GICv5_PPI; + sc->gic_ppi_irqs[irq].gi_isrc.gbi_irq = irq; + sc->gic_ppi_irqs[irq].gi_pol = INTR_POLARITY_CONFORM; + sc->gic_ppi_irqs[irq].gi_trig = INTR_TRIGGER_CONFORM; + isrc = &sc->gic_ppi_irqs[irq].gi_isrc.gbi_isrc; + error = intr_isrc_register(isrc, dev, INTR_ISRCF_PPI, + "%s,p%u", name, irq); + if (error != 0) + panic("%s: Unable to register PPI irq src", + device_get_nameunit(dev)); + } + + /* Assign LPIs to be used as IPIs */ + sc->gic_ipi_irqs = mallocarray((mp_maxid + 1) * INTR_IPI_COUNT, + sizeof(*sc->gic_ipi_irqs), M_DEVBUF, M_ZERO | M_WAITOK); + for (u_int cpu = 0; cpu < (mp_maxid + 1); cpu++) { + struct gicv5_irs *irs; + + irs = NULL; + for (u_int i = 0; i < sc->gic_nirs; i++) { + if (CPU_ISSET(cpu, &sc->gic_irs[i]->irs_cpus)) { + irs = sc->gic_irs[i]; + break; + } + } + if (irs == NULL) + panic("%s: No IRS for CPU %u", + device_get_nameunit(dev), cpu); + + for (u_int ipi = 0; ipi < INTR_IPI_COUNT; ipi++) { + struct intr_irqsrc *isrc; + u_int id = (cpu * INTR_IPI_COUNT) + ipi; + + sc->gic_ipi_irqs[id].gi_irs = irs; + sc->gic_ipi_irqs[id].gi_isrc.gbi_space = GICv5_LPI; + sc->gic_ipi_irqs[id].gi_isrc.gbi_irq = id + + LPI_IPI_BASE; + sc->gic_ipi_irqs[id].gi_pol = INTR_POLARITY_HIGH; + sc->gic_ipi_irqs[id].gi_trig = INTR_TRIGGER_EDGE; + isrc = &sc->gic_ipi_irqs[id].gi_isrc.gbi_isrc; + error = intr_isrc_register(isrc, dev, INTR_ISRCF_IPI, + "%s,cpu%uipi%u", name, cpu, ipi); + if (error != 0) + panic("%s: Unable to register LPI irq src", + device_get_nameunit(dev)); + } + } + + WRITE_SPECIALREG(ICC_PPI_PRIORITYR0_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR1_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR2_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR3_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR4_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR5_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR6_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR7_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR8_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR9_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR10_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR11_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR12_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR13_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR14_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR15_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + + WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, ICC_PPI_ENABLER_NONE); + WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, ICC_PPI_ENABLER_NONE); + isb(); + + /* Set the priority to the lowest value */ + WRITE_SPECIALREG(ICC_PCR_EL1, ICC_PCR_PRIORITY_LOWEST); + + /* Enable interrupts */ + WRITE_SPECIALREG(ICC_CR0_EL1, ICC_CR0_EN); + isb(); +} + +bool +gicv5_add_child(device_t dev, struct gicv5_devinfo *di) +{ + device_t cdev; + struct gicv5_softc *sc; + + cdev = device_add_child(dev, NULL, DEVICE_UNIT_ANY); + if (cdev == NULL) + return (false); + + sc = device_get_softc(dev); + sc->gic_nchildren++; + device_set_ivars(cdev, di); + + return (true); +} + +static int +gicv5_print_child(device_t bus, device_t child) +{ + struct resource_list *rl; + int retval = 0; + + rl = BUS_GET_RESOURCE_LIST(bus, child); + KASSERT(rl != NULL, ("%s: No resource list", __func__)); + retval += bus_print_child_header(bus, child); + retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx"); + retval += bus_print_child_footer(bus, child); + + return (retval); +} + +static u_int +gicv5_lpi_count(struct gicv5_softc *sc) +{ + MPASS(sc->gic_nlpis >= LPI_ITS_BASE); + return (sc->gic_nlpis - LPI_ITS_BASE); +} + +static int +gicv5_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) +{ + struct gicv5_softc *sc; + struct gicv5_devinfo *di; + u_int nlpis; + + switch (which) { + case GICV5_IVAR_LPI_START: + di = device_get_ivars(child); + if (di == NULL || di->di_irs == NULL) + return (EINVAL); + + sc = device_get_softc(dev); + nlpis = gicv5_lpi_count(sc) / sc->gic_nchildren; + *result = LPI_ITS_BASE + device_get_unit(dev) * nlpis; + return (0); + case GICV3_IVAR_NIRQS: + di = device_get_ivars(child); + if (di == NULL || di->di_irs == NULL) + return (EINVAL); + + sc = device_get_softc(dev); + *result = gicv5_lpi_count(sc) / sc->gic_nchildren; + return (0); + case GIC_IVAR_HW_REV: + *result = 5; + return (0); + case GIC_IVAR_BUS: + sc = device_get_softc(dev); + KASSERT(sc->gic_bus != GIC_BUS_UNKNOWN, + ("%s: Unknown bus type", __func__)); + KASSERT(sc->gic_bus <= GIC_BUS_MAX, + ("%s: Invalid bus type %u", __func__, sc->gic_bus)); + *result = sc->gic_bus; + return (0); + case GIC_IVAR_VGIC: + /* TODO when we have vgic support */ + *result = 0; + return (0); + case GIC_IVAR_SUPPORT_LPIS: + di = device_get_ivars(child); + if (di == NULL || di->di_irs == NULL) + return (EINVAL); + + *result = + (IRS_CFG_READ_4(di->di_irs, IRS_IDR2) & IRS_IDR2_LPI) != 0; + return (0); + } + + return (ENOENT); +} + +static int +gicv5_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsize, + cpuset_t *cpuset) +{ + struct gicv5_devinfo *di; + + if (op != LOCAL_CPUS) + return (bus_generic_get_cpus(dev, child, op, setsize, cpuset)); + + di = device_get_ivars(child); + if (di == NULL) + return (bus_generic_get_cpus(dev, child, op, setsize, cpuset)); + + if (setsize != sizeof(cpuset_t)) + return (EINVAL); + + *cpuset = di->di_irs->irs_cpus; + return (0); +} + +static struct resource_list * +gicv5_get_resource_list(device_t bus, device_t child) +{ + struct gicv5_devinfo *di; + + di = device_get_ivars(child); + KASSERT(di != NULL, ("%s: No devinfo", __func__)); + + return (&di->di_rl); +} + +static void +gicv5_eoi(struct gicv5_base_irqsrc *gbi) +{ + uint64_t cddi; + uint32_t irq; + enum gicv5_irq_space space; + + space = gbi->gbi_space; + irq = gbi->gbi_irq; + + /* Drop the priority of the specified interrupt */ + cddi = (uint64_t)space << GIC_CDDI_Type_SHIFT; + cddi |= (uint64_t)irq << GIC_CDDI_ID_SHIFT; + WRITE_SPECIALREG(GIC_CDDI, cddi); + + /* Drop the running priority of the CPU */ + gic_cdeoi(); +} + +int +gicv5_intr(void *arg) +{ + struct gicv5_softc *sc = arg; + struct gicv5_irqsrc *gi; + struct trapframe *tf; + uint64_t hppi; + u_int irq; + + tf = curthread->td_intr_frame; + for (;;) { + hppi = READ_SPECIALREG(GICR_CDIA); + /* Ensure the interrupt activation has completed */ + gsb_ack(); + /* Ensure the gsb ack instruction has completed */ + isb(); + + if ((hppi & ICC_HPPIR_HPPIV) == 0) + return (FILTER_HANDLED); + + irq = (hppi & ICC_HPPIR_ID_MASK) >> ICC_HPPIR_ID_SHIFT; + switch (hppi & ICC_HPPIR_TYPE_MASK) { + case ICC_HPPIR_TYPE_PPI: + MPASS(irq < GICV5_PPI_COUNT); + gi = &sc->gic_ppi_irqs[irq]; + if (intr_isrc_dispatch(&gi->gi_isrc.gbi_isrc, tf) != 0){ + if (gi->gi_trig != INTR_TRIGGER_EDGE) + gicv5_eoi(&gi->gi_isrc); + gicv5_disable_intr(sc->gic_dev, + &gi->gi_isrc.gbi_isrc); + device_printf(sc->gic_dev, + "Stray PPI %u disabled\n", irq); + } + break; + case ICC_HPPIR_TYPE_LPI: + /* XXX */ + if (LPI_IS_IPI(irq)) { + u_int ipi; + + KASSERT(LPI_IPI_IDX(irq) < + ((mp_maxid + 1) * INTR_IPI_COUNT), + ("%s: Invalid IPI LPI %u", __func__, irq)); + ipi = LPI_TO_IPI(irq); + intr_ipi_dispatch(ipi); + gicv5_eoi( + &sc->gic_ipi_irqs[LPI_IPI_IDX(irq)].gi_isrc); + } else { + intr_child_irq_handler(sc->gic_pic, irq); + } + break; + case ICC_HPPIR_TYPE_SPI: + MPASS(irq < sc->gic_spi_count); + gi = &sc->gic_irs_irqs[irq]; + if (intr_isrc_dispatch(&gi->gi_isrc.gbi_isrc, tf) != 0){ + if (gi->gi_trig != INTR_TRIGGER_EDGE) + gicv5_eoi(&gi->gi_isrc); + gicv5_disable_intr(sc->gic_dev, + &gi->gi_isrc.gbi_isrc); + device_printf(sc->gic_dev, + "Stray SPI %u disabled\n", irq); + } + break; + default: + panic("%s: Invalid interrupt type %lx", __func__, + (hppi & ICC_HPPIR_TYPE_MASK) >> + ICC_HPPIR_TYPE_SHIFT); + } + } +} + +static void +gicv5_disable_intr_action(void *argp) +{ + struct gicv5_base_irqsrc *gbi = argp; + uint64_t reg; + uint32_t irq; + + irq = gbi->gbi_irq; + + if (irq < GICV5_PPIS_PER_REG) + reg = READ_SPECIALREG(ICC_PPI_ENABLER0_EL1); + else + reg = READ_SPECIALREG(ICC_PPI_ENABLER1_EL1); + + reg &= ~ICC_PPI_ENABLER_MASK(irq); + reg |= ICC_PPI_ENABLER_DIS(irq); + + if (irq < GICV5_PPIS_PER_REG) + WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, reg); + else + WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, reg); + isb(); +} + +static void +gicv5_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc; + uint32_t irq; + + irq = gbi->gbi_irq; + + switch (gbi->gbi_space) { + case GICv5_PPI: + MPASS((isrc->isrc_flags & INTR_ISRCF_PPI) != 0); + + smp_rendezvous(NULL, gicv5_disable_intr_action, NULL, gbi); + break; + case GICv5_SPI: + WRITE_SPECIALREG(GIC_CDDIS, + GIC_CDDIS_TYPE_SPI | (irq << GIC_CDDIS_ID_SHIFT)); + isb(); + break; + case GICv5_LPI: + WRITE_SPECIALREG(GIC_CDDIS, + GIC_CDDIS_TYPE_LPI | (irq << GIC_CDDIS_ID_SHIFT)); + isb(); + break; + default: + panic("%s: Invalid interrupt space 0x%x", __func__, + gbi->gbi_space); + } +} + +static void +gicv5_enable_intr_action(void *argp) +{ + struct gicv5_base_irqsrc *gbi = argp; + uint64_t reg; + uint32_t irq; + + irq = gbi->gbi_irq; + + if (irq < GICV5_PPIS_PER_REG) + reg = READ_SPECIALREG(ICC_PPI_ENABLER0_EL1); + else + reg = READ_SPECIALREG(ICC_PPI_ENABLER1_EL1); + + reg &= ~ICC_PPI_ENABLER_MASK(irq); + reg |= ICC_PPI_ENABLER_EN(irq); + + if (irq < GICV5_PPIS_PER_REG) + WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, reg); + else + WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, reg); + isb(); +} + +static void +gicv5_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc; + uint32_t irq; + + irq = gbi->gbi_irq; + + switch (gbi->gbi_space) { + case GICv5_PPI: + MPASS((isrc->isrc_flags & INTR_ISRCF_PPI) != 0); + + smp_rendezvous(NULL, gicv5_enable_intr_action, NULL, gbi); + break; + case GICv5_SPI: + WRITE_SPECIALREG(GIC_CDEN, + GIC_CDEN_TYPE_SPI | (irq << GIC_CDEN_ID_SHIFT)); + isb(); + break; + case GICv5_LPI: + WRITE_SPECIALREG(GIC_CDEN, + GIC_CDEN_TYPE_LPI | (irq << GIC_CDEN_ID_SHIFT)); + isb(); + break; + default: + panic("%s: Invalid interrupt space 0x%x", __func__, + gbi->gbi_space); + } +} + +#ifdef FDT +static int +gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, bool *ppi, u_int *irqp, + enum intr_polarity *polp, enum intr_trigger *trigp) +{ + uint64_t reg; + u_int irq; + int type; + + if (ncells < 3) + return (EINVAL); + + /* + * The 1st cell is the interrupt type: + * 1 = PPI + * 2 = LPI + * 3 = SPI + * The 2nd cell contains the interrupt number + * The 3rd cell is the flags, encoded as follows: + * bits[3:0] trigger type and level flags + * 1 = edge triggered + * 2 = edge triggered (PPI only) + * 4 = level-sensitive + * 8 = level-sensitive (PPI only) + */ + switch (cells[0]) { + case 1: + *ppi = true; + break; + /* case 2: LPI */ + case 3: + *ppi = false; + break; + default: + device_printf(dev, "unsupported interrupt type " + "configuration:"); + for (u_int i = 0; i < ncells; i++) + printf(" %x", cells[i]); + printf("\n"); + return (EINVAL); + } + + irq = cells[1]; + + if (ppi) { + /* PPIs are hard coded, ignore cells[2] */ + if (irq < GICV5_PPIS_PER_REG) + reg = READ_SPECIALREG(ICC_PPI_HMR0_EL1); + else + reg = READ_SPECIALREG(ICC_PPI_HMR1_EL1); + + if ((reg & ICC_PPI_HMR_MASK(irq)) == ICC_PPI_HMR_EDGE(irq)) + type = FDT_INTR_EDGE_RISING; + else + type = FDT_INTR_LEVEL_LOW; + } else { + type = cells[2] & FDT_INTR_MASK; + } + + switch (type) { + case FDT_INTR_EDGE_RISING: + *trigp = INTR_TRIGGER_EDGE; + *polp = INTR_POLARITY_HIGH; + break; + case FDT_INTR_EDGE_FALLING: + *trigp = INTR_TRIGGER_EDGE; + *polp = INTR_POLARITY_LOW; + break; + case FDT_INTR_LEVEL_HIGH: + *trigp = INTR_TRIGGER_LEVEL; + *polp = INTR_POLARITY_HIGH; + break; + case FDT_INTR_LEVEL_LOW: + *trigp = INTR_TRIGGER_LEVEL; + *polp = INTR_POLARITY_LOW; + break; + default: + device_printf(dev, "unsupported trigger/polarity " + "configuration 0x%02x\n", cells[2]); + return (EINVAL); + } + + *irqp = irq; + return (0); +} +#endif + +static int +do_gicv5_map_intr(device_t dev, struct intr_map_data *data, bool *ppip, + u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) +{ + struct gicv5_softc *sc; + enum intr_polarity pol; + enum intr_trigger trig; +#ifdef FDT + struct intr_map_data_fdt *daf; +#endif + u_int irq; + bool ppi; + + sc = device_get_softc(dev); + + switch (data->type) { +#ifdef FDT + case INTR_MAP_DATA_FDT: + daf = (struct intr_map_data_fdt *)data; + if (gic_map_fdt(dev, daf->ncells, daf->cells, &ppi, &irq, &pol, + &trig) != 0) + return (EINVAL); + break; +#endif + default: + return (EINVAL); + } + + if (ppi) { + if (irq >= GICV5_PPI_COUNT) + return (EINVAL); + } else { + if (irq > sc->gic_spi_count) + return (EINVAL); + } + switch (pol) { + case INTR_POLARITY_CONFORM: + case INTR_POLARITY_LOW: + case INTR_POLARITY_HIGH: + break; + default: + return (EINVAL); + } + switch (trig) { + case INTR_TRIGGER_CONFORM: + case INTR_TRIGGER_EDGE: + case INTR_TRIGGER_LEVEL: + break; + default: + return (EINVAL); + } + + *ppip = ppi; + *irqp = irq; + if (polp != NULL) + *polp = pol; + if (trigp != NULL) + *trigp = trig; + return (0); +} + +static int +gicv5_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + struct gicv5_softc *sc; + u_int irq; + int error; + bool ppi; + + error = do_gicv5_map_intr(dev, data, &ppi, &irq, NULL, NULL); + if (error == 0) { + sc = device_get_softc(dev); + if (ppi) + *isrcp = &sc->gic_ppi_irqs[irq].gi_isrc.gbi_isrc; + else + *isrcp = &sc->gic_irs_irqs[irq].gi_isrc.gbi_isrc; + } + return (error); +} + +static int +gicv5_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + struct gicv5_irqsrc *gi = (struct gicv5_irqsrc *)isrc; + enum intr_trigger trig; + enum intr_polarity pol; + struct gicv5_irs *irs; + int error; + u_int irq; + bool ppi; + + if (data == NULL) + return (ENOTSUP); + + error = do_gicv5_map_intr(dev, data, &ppi, &irq, &pol, &trig); + if (error != 0) + return (error); + + if (gi->gi_isrc.gbi_irq != irq || pol == INTR_POLARITY_CONFORM || + trig == INTR_TRIGGER_CONFORM) + return (EINVAL); + + if (((isrc->isrc_flags & INTR_ISRCF_PPI) != 0) != ppi) + return (EINVAL); + + /* Compare config if this is not first setup. */ + if (isrc->isrc_handlers != 0) { + if (pol != gi->gi_pol || trig != gi->gi_trig) + return (EINVAL); + else + return (0); + } + + gi->gi_pol = pol; + gi->gi_trig = trig; + + switch (gi->gi_isrc.gbi_space) { + default: + panic("%s: Invalid IRQ space %#x", __func__, + gi->gi_isrc.gbi_space); + case GICv5_PPI: + MPASS(ppi); + MPASS(gi->gi_irs == NULL); + MPASS(irq < GICV5_PPI_COUNT); + /* TODO: Move to smp_rendezvous action */ + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + + break; + case GICv5_SPI: + MPASS(!ppi); + irs = gi->gi_irs; + + /* + * This depends on intr_setup_irq holding the isrc_table_lock + * to serialise access to this register + */ + IRS_CFG_WRITE_4(irs, IRS_SPI_SELR, irq); + error = gicv5_wait_irs_spi_status_idle(irs); + if (error != 0) + return (error); + + /* Set the trigger mode */ + if (trig == INTR_TRIGGER_EDGE) + IRS_CFG_WRITE_4(irs, IRS_SPI_CFGR, + IRS_SPI_CFGR_TM_EDGE); + else + IRS_CFG_WRITE_4(irs, IRS_SPI_CFGR, + IRS_SPI_CFGR_TM_LEVEL); + + error = gicv5_wait_irs_spi_status_idle(irs); + return (error); + } + + return (0); +} + +static int +gicv5_teardown_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + return (0); +} + +static void +gicv5_post_filter(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc; + + gicv5_eoi(gbi); +} + +static void +gicv5_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc; + + switch (gbi->gbi_space) { + case GICv5_PPI: + gicv5_disable_intr_action(isrc); + break; + default: + gicv5_disable_intr(dev, isrc); + break; + } + gicv5_eoi(gbi); +} + +static void +gicv5_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc; + + switch (gbi->gbi_space) { + case GICv5_PPI: + gicv5_enable_intr_action(isrc); + break; + default: + gicv5_enable_intr(dev, isrc); + break; + } +} + +static int +gicv5_bind_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv5_irqsrc *gi = (struct gicv5_irqsrc *)isrc; + struct gicv5_base_irqsrc *gbi = &gi->gi_isrc; + struct gicv5_irs *irs; + uint64_t cdaff, cdpri; + uint32_t irq; + int cpu, iaffid; + + if (CPU_EMPTY(&isrc->isrc_cpu)) { + irs = gi->gi_irs; + cpu = irs->irs_next_irq_cpu = + intr_irq_next_cpu(irs->irs_next_irq_cpu, &irs->irs_cpus); + } else { + cpu = CPU_FFS(&isrc->isrc_cpu) - 1; + } + + MPASS(cpu <= mp_maxid); + iaffid = gicv5_iaffids[cpu]; + MPASS(iaffid >= 0); + + irq = gbi->gbi_irq; + + /* TODO: Where should priority be set? */ + cdpri = GIC_CDPRI_PRORITY(GICV5_PRI_LOWEST); + cdpri |= GIC_CDPRI_ID(irq); + + cdaff = GIC_CDAFF_IAFFID(iaffid); + cdaff |= GIC_CDAFF_IRM_TARGETED; + cdaff |= GIC_CDAFF_ID(irq); + + switch (gbi->gbi_space) { + default: + panic("%s: Invalid space %x", __func__, gbi->gbi_space); + case GICv5_SPI: + cdpri |= GIC_CDPRI_TYPE_SPI; + cdaff |= GIC_CDAFF_TYPE_SPI; + break; + case GICv5_LPI: + cdpri |= GIC_CDPRI_TYPE_LPI; + cdaff |= GIC_CDAFF_TYPE_LPI; + break; + } + + WRITE_SPECIALREG(GIC_CDPRI, cdpri); + WRITE_SPECIALREG(GIC_CDAFF, cdaff); + isb(); + + return (0); +} + +#ifdef SMP +static void +gicv5_init_secondary(device_t dev, uint32_t rootnum) +{ + struct gicv5_softc *sc; + u_int cpu; + + sc = device_get_softc(dev); + cpu = curcpu; + + WRITE_SPECIALREG(ICC_PPI_PRIORITYR0_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR1_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR2_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR3_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR4_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR5_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR6_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR7_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR8_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR9_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR10_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR11_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR12_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR13_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR14_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + WRITE_SPECIALREG(ICC_PPI_PRIORITYR15_EL1, + ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST)); + + /* Disable all PPIs, then enable as needed */ + WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, ICC_PPI_ENABLER_NONE); + WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, ICC_PPI_ENABLER_NONE); + isb(); + + /* Set the priority to the lowest value */ + WRITE_SPECIALREG(ICC_PCR_EL1, ICC_PCR_PRIORITY_LOWEST); + + /* Enable interrupts */ + WRITE_SPECIALREG(ICC_CR0_EL1, ICC_CR0_EN); + isb(); + + for (u_int irq = 0; irq < GICV5_PPI_COUNT; irq++) { + struct intr_irqsrc *isrc; + + isrc = &sc->gic_ppi_irqs[irq].gi_isrc.gbi_isrc; + if (intr_isrc_init_on_cpu(isrc, cpu)) { + gicv5_enable_intr_action(isrc); + } + } + + /* Set the priority to the lowest value */ + WRITE_SPECIALREG(ICC_PCR_EL1, ICC_PCR_PRIORITY_LOWEST); + + /* Enable interrupts */ + WRITE_SPECIALREG(ICC_CR0_EL1, ICC_CR0_EN); + isb(); +} + +static void +gicv5_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, + u_int ipi) +{ + struct gicv5_irqsrc *gi = (struct gicv5_irqsrc *)isrc; + uint64_t val; + + val = GIC_CDPEND_ID(gi->gi_isrc.gbi_irq); + switch (gi->gi_isrc.gbi_space) { + default: + panic("%s: Invalid space: %x", __func__, gi->gi_isrc.gbi_space); + case GICv5_LPI: + val |= GIC_CDPEND_TYPE_LPI; + break; + } + val |= GIC_CDPEND_PENDING_SET; + WRITE_SPECIALREG(GIC_CDPEND, val); +} + +static int +gicv5_ipi_setup_per_cpu(device_t dev, u_int ipi, struct intr_irqsrc **isrcp, + u_int cpu) +{ + struct gicv5_softc *sc; + struct gicv5_irqsrc *gi; + u_int id, irq; + int iaffid; + + sc = device_get_softc(dev); + iaffid = gicv5_iaffids[cpu]; + KASSERT(iaffid >= 0, ("%s: No iaffid for cpu %u", __func__, cpu)); + id = (cpu * INTR_IPI_COUNT) + ipi; + gi = &sc->gic_ipi_irqs[id]; + CPU_SET(cpu, &gi->gi_isrc.gbi_isrc.isrc_cpu); + + irq = gi->gi_isrc.gbi_irq; + WRITE_SPECIALREG(GIC_CDPRI, GIC_CDPRI_PRORITY(GICV5_PRI_LOWEST) | + GIC_CDPRI_TYPE_LPI | GIC_CDPRI_ID(irq)); + WRITE_SPECIALREG(GIC_CDAFF, GIC_CDAFF_IAFFID(iaffid) | + GIC_CDAFF_TYPE_LPI | GIC_CDAFF_ID(irq)); + + *isrcp = &gi->gi_isrc.gbi_isrc; + return (0); +} + +static cpu_feat_en +gicv5_feat_check(const struct cpu_feat *feat __unused, u_int midr __unused) +{ + if (gicv5_iaffids == NULL) + return (FEAT_ALWAYS_DISABLE); + + return (FEAT_ALWAYS_ENABLE); +} + +static bool +gicv5_feat_enable(const struct cpu_feat *feat __unused, + cpu_feat_errata errata_status __unused, u_int *errata_list __unused, + u_int errata_count __unused) +{ + u_int cpu; + + /* This is handled by attach */ + cpu = curcpu; + if (cpu == 0) + return (true); + + MPASS(cpu <= mp_maxid); + MPASS(gicv5_iaffids[cpu] == -1); + gicv5_iaffids[cpu] = + ICC_IAFFIDR_IAFFID_VAL(READ_SPECIALREG(ICC_IAFFIDR_EL1)); + + return (true); +} + +CPU_FEAT(gicv5, "GICv5", + gicv5_feat_check, NULL, gicv5_feat_enable, NULL, + CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); +#endif diff --git a/sys/arm64/arm64/gicv5_fdt.c b/sys/arm64/arm64/gicv5_fdt.c new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gicv5_fdt.c @@ -0,0 +1,298 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * Copyright (c) 2025 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include "gicv5var.h" + +#include "pci_if.h" + +struct gicv5_fdt_devinfo { + struct gicv5_devinfo di_base; + struct ofw_bus_devinfo di_dinfo; +}; + +static device_probe_t gicv5_fdt_probe; +static device_probe_t gicv5_fdt_attach; + +static ofw_bus_get_devinfo_t gicv5_fdt_get_devinfo; + +static device_method_t gicv5_fdt_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, gicv5_fdt_probe), + DEVMETHOD(device_attach, gicv5_fdt_attach), + + /* ofw_bus interface */ + DEVMETHOD(ofw_bus_get_devinfo, gicv5_fdt_get_devinfo), + DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), + DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), + DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), + DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), + DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_1(gic, gicv5_fdt_driver, gicv5_fdt_methods, + sizeof(struct gicv5_softc), gicv5_driver); + +EARLY_DRIVER_MODULE(gicv5, simplebus, gicv5_fdt_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); + +static int +gicv5_fdt_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "arm,gic-v5")) + return (ENXIO); + + device_set_desc(dev, "ARM Generic Interrupt Controller v5"); + return (BUS_PROBE_DEFAULT); +} + +static void +gicv5_fdt_add_child(device_t dev, phandle_t node, struct gicv5_irs *irs, + pcell_t addr_cells, pcell_t size_cells) +{ + struct gicv5_fdt_devinfo *di; + + di = malloc(sizeof(*di), M_DEVBUF, M_WAITOK | M_ZERO); + di->di_base.di_irs = irs; + if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node)) { + if (bootverbose) { + device_printf(dev, + "Could not set up devinfo\n"); + } + free(di, M_DEVBUF); + return; + } + + /* Initialize and populate resource list. */ + resource_list_init(&di->di_base.di_rl); + ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells, + &di->di_base.di_rl); + + if (!gicv5_add_child(dev, &di->di_base)) { + if (bootverbose) { + device_printf(dev, + "Could not add child: %s\n", + di->di_dinfo.obd_name); + } + resource_list_free(&di->di_base.di_rl); + ofw_bus_gen_destroy_devinfo(&di->di_dinfo); + free(di, M_DEVBUF); + } +} + +static void +gicv5_fdt_add_irs_children(device_t dev, struct gicv5_irs *irs, + phandle_t node, pcell_t addr_cells, pcell_t size_cells) +{ + phandle_t child; + + for (child = OF_child(node); child != 0; child = OF_peer(child)) + gicv5_fdt_add_child(dev, child, irs, addr_cells, size_cells); +} + +static void +gicv5_fdt_add_children(device_t dev) +{ + char compat[16]; + struct gicv5_softc *sc; + phandle_t node; + pcell_t addr_cells, size_cells; + phandle_t child; + u_int irs_idx; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + if (OF_getencprop(node, "#address-cells", &addr_cells, + sizeof(addr_cells)) == -1) { + device_printf(dev, "Unable to read #address-cells\n"); + return; + } + if (OF_getencprop(node, "#size-cells", &size_cells, + sizeof(size_cells)) == -1) { + device_printf(dev, "Unable to read #size-cells\n"); + return; + } + + irs_idx = 0; + /* Find children nodes to attach devices to */ + for (child = OF_child(node); child != 0; child = OF_peer(child)) { + if (OF_getprop(child, "compatible", compat, sizeof(compat)) < 0) + continue; + + /* Don't connect to the IRS, but connect to its children */ + if (strcmp(compat, "arm,gic-v5-irs") == 0) { + MPASS(irs_idx < sc->gic_nirs); + gicv5_fdt_add_irs_children(dev, sc->gic_irs[irs_idx], + child, addr_cells, size_cells); + irs_idx++; + continue; + } + + gicv5_fdt_add_child(dev, child, NULL, addr_cells, size_cells); + } +} + +static int +gicv5_fdt_attach(device_t dev) +{ + char compat[16]; + struct gicv5_softc *sc; + ssize_t rv; + phandle_t child, node; + bus_addr_t paddr; + bus_size_t size; + intptr_t xref; + int error, i; + + sc = device_get_softc(dev); + sc->gic_bus = GIC_BUS_FDT; + + node = ofw_bus_get_node(dev); + for (child = OF_child(node); child != 0; child = OF_peer(child)) { + if (OF_getprop(child, "compatible", compat, sizeof(compat)) < 0) + continue; + + if (strcmp(compat, "arm,gic-v5-irs") != 0) + continue; + + sc->gic_nirs++; + } + if (sc->gic_nirs == 0) + panic("%s: Invalid configuration, no IRS defined", + device_get_nameunit(dev)); + sc->gic_irs = mallocarray(sc->gic_nirs, sizeof(sc->gic_irs[0]), + M_DEVBUF, M_WAITOK | M_ZERO); + i = 0; + for (child = OF_child(node); child != 0; child = OF_peer(child)) { + cpuset_t cpuset; + pcell_t *cpus; + + if (OF_getprop(child, "compatible", compat, sizeof(compat)) < 0) + continue; + + if (strcmp(compat, "arm,gic-v5-irs") != 0) + continue; + + /* TODO: What to do with a standalone IRS? */ + rv = OF_getencprop_alloc(child, "cpus", (void **)&cpus); + if (rv < 0) + continue; + + CPU_ZERO(&cpuset); + for (u_int c = 0; c < rv / sizeof(pcell_t); c++) { + struct pcpu *pcpu; + device_t cpu; + uintptr_t v; + + cpu = OF_device_from_xref(cpus[c]); + if (cpu == NULL) { + device_printf(dev, + "Unable to find device for CPU %u\n", c); + continue; + } + + /* + * We can't use cpu_get_pcpu here as it expect the + * passed in device to be a direct child of the cpu + * device. + */ + if (BUS_READ_IVAR(cpu, dev, CPU_IVAR_PCPU, &v) != 0 || + v == 0) { + device_printf(dev, "No PCPU for %s\n", + device_get_nameunit(cpu)); + continue; + } + pcpu = (struct pcpu *)v; + + CPU_SET(pcpu->pc_cpuid, &cpuset); + } + + OF_prop_free(cpus); + + gicv5_irs_init(dev, i, &cpuset); + error = ofw_reg_to_paddr(child, 0, &paddr, &size, NULL); + if (error != 0) + panic("%s: Invalid configuration, no physical address " + "or size found for child irs %u", + device_get_nameunit(dev), i); + + error = bus_set_resource(dev, SYS_RES_MEMORY, i, paddr, size); + if (error != 0) + panic("%s: Unable to set memory resource", + device_get_nameunit(dev)); + i++; + } + + gicv5_attach(dev); + + xref = OF_xref_from_node(node); + sc->gic_pic = intr_pic_register(dev, xref); + if (sc->gic_pic == NULL) + panic("%s: Unable to register PIC", device_get_nameunit(dev)); + + /* Register xref */ + OF_device_register_xref(xref, dev); + intr_ipi_pic_register_flags(dev, 0, INTR_IPI_PER_CPU); + + error = intr_pic_claim_root(dev, xref, gicv5_intr, sc, INTR_ROOT_IRQ); + if (error != 0) + panic("%s: Unable to claim PIC root", device_get_nameunit(dev)); + + gicv5_fdt_add_children(dev); + bus_attach_children(dev); + + return (0); +} + +static const struct ofw_bus_devinfo * +gicv5_fdt_get_devinfo(device_t dev __unused, device_t child) +{ + struct gicv5_fdt_devinfo *di; + + di = device_get_ivars(child); + return (&di->di_dinfo); +} diff --git a/sys/arm64/arm64/gicv5reg.h b/sys/arm64/arm64/gicv5reg.h new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gicv5reg.h @@ -0,0 +1,707 @@ +/* + * Copyright (c) 2025 Arm Ltd + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _ARM64_GICV5REG_H_ +#define _ARM64_GICV5REG_H_ + +#include + +/* + * GIC instructions in the SYS instruction space + */ + +/* + * GIC CDAFF, + * Interrupt Set Target in the Current Interrupt Domain + */ +#define GIC_CDAFF MRS_REG_ALT_NAME(GIC_CDAFF) +#define GIC_CDAFF_op0 1 +#define GIC_CDAFF_op1 0 +#define GIC_CDAFF_CRn 12 +#define GIC_CDAFF_CRm 1 +#define GIC_CDAFF_op2 3 +#define GIC_CDAFF_IAFFID_SHIFT 32 +#define GIC_CDAFF_IAFFID_MASK (0xfffful << GIC_CDAFF_IAFFID_SHIFT) +#define GIC_CDAFF_IAFFID(x) \ + ((uint64_t)(x) << GIC_CDAFF_IAFFID_SHIFT) +#define GIC_CDAFF_TYPE_SHIFT 29 +#define GIC_CDAFF_TYPE_MASK (0x7ul << GIC_CDAFF_TYPE_SHIFT) +#define GIC_CDAFF_TYPE_LPI (0x2ul << GIC_CDAFF_TYPE_SHIFT) +#define GIC_CDAFF_TYPE_SPI (0x3ul << GIC_CDAFF_TYPE_SHIFT) +#define GIC_CDAFF_IRM_SHIFT 28 +#define GIC_CDAFF_IRM_MASK (0x1ul << GIC_CDAFF_IRM_SHIFT) +#define GIC_CDAFF_IRM_TARGETED (0x0ul << GIC_CDAFF_IRM_SHIFT) +#define GIC_CDAFF_IRM_1ofN (0x1ul << GIC_CDAFF_IRM_SHIFT) +#define GIC_CDAFF_ID_SHIFT 0 +#define GIC_CDAFF_ID_MASK (0xfffffful << GIC_CDAFF_ID_SHIFT) +#define GIC_CDAFF_ID(x) ((uint64_t)(x) << GIC_CDAFF_ID_SHIFT) + +/* + * GIC CDDI, + * Interrupt Deactivate in the Current Interrupt Domain + */ +#define GIC_CDDI MRS_REG_ALT_NAME(GIC_CDDI) +#define GIC_CDDI_op0 1 +#define GIC_CDDI_op1 0 +#define GIC_CDDI_CRn 12 +#define GIC_CDDI_CRm 2 +#define GIC_CDDI_op2 0 +#define GIC_CDDI_Type_SHIFT 29 +#define GIC_CDDI_Type_MASK (0x7ul << GIC_CDDI_Type_SHIFT) +#define GIC_CDDI_Type_PPI (0x1ul << GIC_CDDI_Type_SHIFT) +#define GIC_CDDI_Type_LPI (0x2ul << GIC_CDDI_Type_SHIFT) +#define GIC_CDDI_Type_SPI (0x3ul << GIC_CDDI_Type_SHIFT) +#define GIC_CDDI_ID_SHIFT 0 +#define GIC_CDDI_ID_MASK (0xfffffful << GIC_CDDI_ID_SHIFT) + +/* + * GIC CDDIS, + * Interrupt Disable in the Current Interrupt Domain + */ +#define GIC_CDDIS MRS_REG_ALT_NAME(GIC_CDDIS) +#define GIC_CDDIS_op0 1 +#define GIC_CDDIS_op1 0 +#define GIC_CDDIS_CRn 12 +#define GIC_CDDIS_CRm 1 +#define GIC_CDDIS_op2 0 +#define GIC_CDDIS_TYPE_SHIFT 29 +#define GIC_CDDIS_TYPE_WIDTH 3 +#define GIC_CDDIS_TYPE_MASK (UL(0x7) << GIC_CDDIS_TYPE_SHIFT) +#define GIC_CDDIS_TYPE_LPI (UL(0x2) << GIC_CDDIS_TYPE_SHIFT) +#define GIC_CDDIS_TYPE_SPI (UL(0x3) << GIC_CDDIS_TYPE_SHIFT) +#define GIC_CDDIS_ID_SHIFT 0 +#define GIC_CDDIS_ID_WIDTH 24 +#define GIC_CDDIS_ID_MASK (UL(0xffffff) << GIC_CDDIS_ID_SHIFT) + +/* + * GIC CDEN, + * Interrupt Enable in the Current Interrupt Domain + */ +#define GIC_CDEN MRS_REG_ALT_NAME(GIC_CDEN) +#define GIC_CDEN_op0 1 +#define GIC_CDEN_op1 0 +#define GIC_CDEN_CRn 12 +#define GIC_CDEN_CRm 1 +#define GIC_CDEN_op2 1 +#define GIC_CDEN_TYPE_SHIFT 29 +#define GIC_CDEN_TYPE_WIDTH 3 +#define GIC_CDEN_TYPE_MASK (UL(0x7) << GIC_CDEN_TYPE_SHIFT) +#define GIC_CDEN_TYPE_LPI (UL(0x2) << GIC_CDEN_TYPE_SHIFT) +#define GIC_CDEN_TYPE_SPI (UL(0x3) << GIC_CDEN_TYPE_SHIFT) +#define GIC_CDEN_ID_SHIFT 0 +#define GIC_CDEN_ID_WIDTH 24 +#define GIC_CDEN_ID_MASK (UL(0xffffff) << GIC_CDEN_ID_SHIFT) + +/* + * GIC CDEOI, + * Priority Drop in the Current Interrupt Domain + */ +#define GIC_CDEOI MRS_REG_ALT_NAME(GIC_CDEOI) +#define GIC_CDEOI_op0 1 +#define GIC_CDEOI_op1 0 +#define GIC_CDEOI_CRn 12 +#define GIC_CDEOI_CRm 1 +#define GIC_CDEOI_op2 7 +#define gic_cdeoi() \ + __asm __volatile("msr " __XSTRING(GIC_CDEOI) ", xzr" ::: "memory") + +/* + * GIC CDHM, + * Interrupt Handling mode state in the Current Interrupt Domain + */ +#define GIC_CDHM MRS_REG_ALT_NAME(GIC_CDHM) +#define GIC_CDHM_op0 1 +#define GIC_CDHM_op1 0 +#define GIC_CDHM_CRn 12 +#define GIC_CDHM_CRm 2 +#define GIC_CDHM_op2 1 +#define GIC_CDHM_HM_MASK (0x1ul << 32) +#define GIC_CDHM_HM_EDGE (0x0ul << 32) +#define GIC_CDHM_HM_LEVEL (0x1ul << 32) +#define GIC_CDHM_TYPE_SHIFT 29 +#define GIC_CDHM_TYPE_MASK (0x7ul << GIC_CDHM_TYPE_SHIFT) +#define GIC_CDHM_TYPE_PPI (0x1ul << GIC_CDHM_TYPE_SHIFT) +#define GIC_CDHM_TYPE_LPI (0x2ul << GIC_CDHM_TYPE_SHIFT) +#define GIC_CDHM_TYPE_SPI (0x3ul << GIC_CDHM_TYPE_SHIFT) +#define GIC_CDHM_ID_SHIFT 0 +#define GIC_CDHM_ID_MASK (0xfffffful << GIC_CDHM_ID_SHIFT) + +/* + * GIC CDPEND, + * Interrupt Set/Clear Pending state in the Current Interrupt Domain + */ +#define GIC_CDPEND MRS_REG_ALT_NAME(GIC_CDPEND) +#define GIC_CDPEND_op0 1 +#define GIC_CDPEND_op1 0 +#define GIC_CDPEND_CRn 12 +#define GIC_CDPEND_CRm 1 +#define GIC_CDPEND_op2 4 +#define GIC_CDPEND_PENDING_SHIFT 32 +#define GIC_CDPEND_PENDING_MASK (0x1ul << GIC_CDPEND_PENDING_SHIFT) +#define GIC_CDPEND_PENDING_CLEAR (0x0ul << GIC_CDPEND_PENDING_SHIFT) +#define GIC_CDPEND_PENDING_SET (0x1ul << GIC_CDPEND_PENDING_SHIFT) +#define GIC_CDPEND_TYPE_SHIFT 29 +#define GIC_CDPEND_TYPE_MASK (0x7ul << GIC_CDPEND_TYPE_SHIFT) +#define GIC_CDPEND_TYPE_LPI (0x2ul << GIC_CDPEND_TYPE_SHIFT) +#define GIC_CDPEND_TYPE_SPI (0x2ul << GIC_CDPEND_TYPE_SHIFT) +#define GIC_CDPEND_ID_SHIFT 0 +#define GIC_CDPEND_ID_MASK (0xfffffful << GIC_CDPEND_ID_SHIFT) +#define GIC_CDPEND_ID(x) ((uint64_t)(x) << GIC_CDPEND_ID_SHIFT) + +/* + * GIC CDPRI, + * Interrupt Set priority in the Current Interrupt Domain + */ +#define GIC_CDPRI MRS_REG_ALT_NAME(GIC_CDPRI) +#define GIC_CDPRI_op0 1 +#define GIC_CDPRI_op1 0 +#define GIC_CDPRI_CRn 12 +#define GIC_CDPRI_CRm 1 +#define GIC_CDPRI_op2 2 +#define GIC_CDPRI_PRORITY_SHIFT 35 +#define GIC_CDPRI_PRORITY_MASK (0x1ful << GIC_CDPRI_PRORITY_SHIFT) +#define GIC_CDPRI_PRORITY(x) \ + ((uint64_t)(x) << GIC_CDPRI_PRORITY_SHIFT) +#define GIC_CDPRI_TYPE_SHIFT 29 +#define GIC_CDPRI_TYPE_MASK (0x7ul << GIC_CDPRI_TYPE_SHIFT) +#define GIC_CDPRI_TYPE_LPI (0x2ul << GIC_CDPRI_TYPE_SHIFT) +#define GIC_CDPRI_TYPE_SPI (0x3ul << GIC_CDPRI_TYPE_SHIFT) +#define GIC_CDPRI_ID_SHIFT 0 +#define GIC_CDPRI_ID_MASK (0xfffffful << GIC_CDPRI_ID_SHIFT) +#define GIC_CDPRI_ID(x) ((uint64_t)(x) << GIC_CDPRI_ID_SHIFT) + +/* + * GIC CDRCFG, + * Interrupt Set priority in the Current Interrupt Domain + */ +#define GIC_CDRCFG MRS_REG_ALT_NAME(GIC_CDRCFG) +#define GIC_CDRCFG_op0 1 +#define GIC_CDRCFG_op1 0 +#define GIC_CDRCFG_CRn 12 +#define GIC_CDRCFG_CRm 1 +#define GIC_CDRCFG_op2 5 + +/* + * GICR , CDIA + * Interrupt Acknowledge in the Current Interrupt Domain + */ +#define GICR_CDIA MRS_REG_ALT_NAME(GICR_CDIA) +#define GICR_CDIA_op0 1 +#define GICR_CDIA_op1 0 +#define GICR_CDIA_CRn 12 +#define GICR_CDIA_CRm 3 +#define GICR_CDIA_op2 0 + + +/* + * GICR , CDNMIA + * Non-maskable Interrupt Acknowledge in the Current Interrupt Domain + */ +#define GICR_CDNMIA MRS_REG_ALT_NAME(GICR_CDNMIA) +#define GICR_CDNMIA_op0 1 +#define GICR_CDNMIA_op1 0 +#define GICR_CDNMIA_CRn 12 +#define GICR_CDNMIA_CRm 3 +#define GICR_CDNMIA_op2 1 + +/* + * GSB ACK + * GIC Synchronization Barrier Interrupt Acknowledge + */ +#define GSB_ACK MRS_REG_ALT_NAME(GSB_ACK) +#define GSB_ACK_op0 1 +#define GSB_ACK_op1 0 +#define GSB_ACK_CRn 12 +#define GSB_ACK_CRm 0 +#define GSB_ACK_op2 1 +#define gsb_ack() \ + __asm __volatile("msr " __XSTRING(GSB_ACK) ", xzr" ::: "memory") + +/* + * GSB SYS + * GIC Synchronization Barrier System + */ +#define GSB_SYS MRS_REG_ALT_NAME(GSB_SYS) +#define GSB_SYS_op0 1 +#define GSB_SYS_op1 0 +#define GSB_SYS_CRn 12 +#define GSB_SYS_CRm 0 +#define GSB_SYS_op2 0 +#define gsb_sys() \ + __asm __volatile("msr " __XSTRING(GSB_SYS) ", xzr" ::: "memory") + + +/* + * CPU interface registers + */ + +/* + * ICC_APR_EL1 + * Interrupt Controller Physical Active Priorities Register + */ +#define ICC_APR_EL1 MRS_REG_ALT_NAME(ICC_APR_EL1) +#define ICC_APR_EL1_op0 3 +#define ICC_APR_EL1_op1 1 +#define ICC_APR_EL1_CRn 12 +#define ICC_APR_EL1_CRm 0 +#define ICC_APR_EL1_op2 0 +#define ICC_APR_P(x) (1ul << (x)) + +/* + * ICC_CR0_EL1 + * Interrupt Controller EL1 Physical Control Register + */ +#define ICC_CR0_EL1 MRS_REG_ALT_NAME(ICC_CR0_EL1) +#define ICC_CR0_EL1_op0 3 +#define ICC_CR0_EL1_op1 1 +#define ICC_CR0_EL1_CRn 12 +#define ICC_CR0_EL1_CRm 0 +#define ICC_CR0_EL1_op2 1 +#define ICC_CR0_PID (0x1ul << 38) +#define ICC_CR0_IPPT_SHIFT 32 +#define ICC_CR0_IPPT_MASK (0x1ful << ICC_CR0_IPPT_SHIFT) +#define ICC_CR0_EN (0x1ul << 0) + +/* + * ICC_HAPR_EL1 + * Interrupt Controller Physical Highest Active Priority Register + */ +#define ICC_HAPR_EL1 MRS_REG_ALT_NAME(ICC_HAPR_EL1) +#define ICC_HAPR_EL1_op0 3 +#define ICC_HAPR_EL1_op1 1 +#define ICC_HAPR_EL1_CRn 12 +#define ICC_HAPR_EL1_CRm 0 +#define ICC_HAPR_EL1_op2 3 +#define ICC_HAPR_PRIORITY_SHIFT 0 +#define ICC_HAPR_PRIORITY_MASK (0xfful << ICC_HAPR_PRIORITY_SHIFT) + +/* + * ICC_HPPIR_EL1 + * Interrupt Controller Physical Highest Priority Pending Interrupt Register + */ +#define ICC_HPPIR_EL1 MRS_REG_ALT_NAME(ICC_HPPIR_EL1) +#define ICC_HPPIR_EL1_op0 3 +#define ICC_HPPIR_EL1_op1 0 +#define ICC_HPPIR_EL1_CRn 12 +#define ICC_HPPIR_EL1_CRm 10 +#define ICC_HPPIR_EL1_op2 3 +#define ICC_HPPIR_HPPIV (0x1ul << 32) +#define ICC_HPPIR_TYPE_SHIFT 29 +#define ICC_HPPIR_TYPE_MASK (0x7ul << ICC_HPPIR_TYPE_SHIFT) +#define ICC_HPPIR_TYPE_PPI (0x1ul << ICC_HPPIR_TYPE_SHIFT) +#define ICC_HPPIR_TYPE_LPI (0x2ul << ICC_HPPIR_TYPE_SHIFT) +#define ICC_HPPIR_TYPE_SPI (0x3ul << ICC_HPPIR_TYPE_SHIFT) +#define ICC_HPPIR_ID_SHIFT 0 +#define ICC_HPPIR_ID_MASK (0xfffffful << ICC_HPPIR_ID_SHIFT) + +/* + * ICC_IAFFID_EL1 + * Interrupt Controller PE Interrupt Affinity ID Register + */ +#define ICC_IAFFIDR_EL1 MRS_REG_ALT_NAME(ICC_IAFFIDR_EL1) +#define ICC_IAFFIDR_EL1_op0 3 +#define ICC_IAFFIDR_EL1_op1 0 +#define ICC_IAFFIDR_EL1_CRn 12 +#define ICC_IAFFIDR_EL1_CRm 10 +#define ICC_IAFFIDR_EL1_op2 5 +#define ICC_IAFFIDR_IAFFID_SHIFT 0 +#define ICC_IAFFIDR_IAFFID_MASK (0xfffful << ICC_IAFFIDR_IAFFID_SHIFT) +#define ICC_IAFFIDR_IAFFID_VAL(x) \ + (((x) & ICC_IAFFIDR_IAFFID_MASK) >> ICC_IAFFIDR_IAFFID_SHIFT) + +/* + * ICC_ICSR_EL1 + * Interrupt Controller Interrupt Configuration and State Register + */ +#define ICC_ICSR_EL1 MRS_REG_ALT_NAME(ICC_ICSR_EL1) +#define ICC_ICSR_EL1_op0 3 +#define ICC_ICSR_EL1_op1 0 +#define ICC_ICSR_EL1_CRn 12 +#define ICC_ICSR_EL1_CRm 10 +#define ICC_ICSR_EL1_op2 4 +#define ICC_ICSR_IAFFID_SHIFT 32 +#define ICC_ICSR_IAFFID_MASK (0xfffful << ICC_ICSR_IAFFID_SHIFT) +#define ICC_ICSR_Priority_SHIFT 11 +#define ICC_ICSR_Priority_MASK (0x1ful << ICC_ICSR_Priority_SHIFT) +#define ICC_ICSR_HM (0x1ul << 5) +#define ICC_ICSR_Active (0x1ul << 4) +#define ICC_ICSR_IRM (0x1ul << 3) +#define ICC_ICSR_Pending (0x1ul << 2) +#define ICC_ICSR_Enabled (0x1ul << 1) +#define ICC_ICSR_F (0x1ul << 0) + +/* + * ICC_IDR0_EL1 + * Interrupt Controller ID Register 0 + */ +#define ICC_IDR0_EL1 MRS_REG_ALT_NAME(ICC_IDR0_EL1) +#define ICC_IDR0_EL1_op0 3 +#define ICC_IDR0_EL1_op1 0 +#define ICC_IDR0_EL1_CRn 12 +#define ICC_IDR0_EL1_CRm 10 +#define ICC_IDR0_EL1_op2 2 +#define ICC_IDR0_GCIE_LEGACY_SHIFT 8 +#define ICC_IDR0_GCIE_LEGACY_MASK (0xful << ICC_IDR0_GCIE_LEGACY_SHIFT) +#define ICC_IDR0_PRI_BITS_SHIFT 4 +#define ICC_IDR0_PRI_BITS_MASK (0xful << ICC_IDR0_PRI_BITS_SHIFT) +#define ICC_IDR0_PRI_BITS_4 (0x3ul << ICC_IDR0_PRI_BITS_SHIFT) +#define ICC_IDR0_PRI_BITS_5 (0x4ul << ICC_IDR0_PRI_BITS_SHIFT) +#define ICC_IDR0_ID_BITS_SHIFT 0 +#define ICC_IDR0_ID_BITS_MASK (0xful << ICC_IDR0_ID_BITS_SHIFT) +#define ICC_IDR0_ID_BITS_16 (0x0ul << ICC_IDR0_ID_BITS_SHIFT) +#define ICC_IDR0_ID_BITS_24 (0x1ul << ICC_IDR0_ID_BITS_SHIFT) + +/* + * ICC_PCR_EL1 + * Interrupt Controller Physical Interrupt Priority Control Register + * This register is self-synchronizing + */ +#define ICC_PCR_EL1 MRS_REG_ALT_NAME(ICC_PCR_EL1) +#define ICC_PCR_EL1_op0 3 +#define ICC_PCR_EL1_op1 1 +#define ICC_PCR_EL1_CRn 12 +#define ICC_PCR_EL1_CRm 0 +#define ICC_PCR_EL1_op2 2 +#define ICC_PCR_PRIORITY_SHIFT 0 +#define ICC_PCR_PRIORITY_MASK (0x1ful << ICC_PCR_PRIORITY_SHIFT) +#define ICC_PCR_PRIORITY_LOWEST (0x1ful << ICC_PCR_PRIORITY_SHIFT) + +/* + * PPI registers + */ + +/* + * ICC_PPI_CACTIVER_EL1 + * Interrupt Controller Physical PPI Clear Active Register + */ +#define ICC_PPI_CACTIVER0_EL1 MRS_REG_ALT_NAME(ICC_PPI_CACTIVER0_EL1) +#define ICC_PPI_CACTIVER0_EL1_op0 3 +#define ICC_PPI_CACTIVER0_EL1_op1 0 +#define ICC_PPI_CACTIVER0_EL1_CRn 12 +#define ICC_PPI_CACTIVER0_EL1_CRm 13 +#define ICC_PPI_CACTIVER0_EL1_op2 0 + +#define ICC_PPI_CACTIVER1_EL1 MRS_REG_ALT_NAME(ICC_PPI_CACTIVER1_EL1) +#define ICC_PPI_CACTIVER1_EL1_op0 3 +#define ICC_PPI_CACTIVER1_EL1_op1 0 +#define ICC_PPI_CACTIVER1_EL1_CRn 12 +#define ICC_PPI_CACTIVER1_EL1_CRm 13 +#define ICC_PPI_CACTIVER1_EL1_op2 1 + +/* + * ICC_PPI_CPENDR_EL1 + * Interrupt Controller Physical PPI Clear Pending State Registers + */ +#define ICC_PPI_CPENDR0_EL1 MRS_REG_ALT_NAME(ICC_PPI_CPENDR0_EL1) +#define ICC_PPI_CPENDR0_EL1_op0 3 +#define ICC_PPI_CPENDR0_EL1_op1 0 +#define ICC_PPI_CPENDR0_EL1_CRn 12 +#define ICC_PPI_CPENDR0_EL1_CRm 13 +#define ICC_PPI_CPENDR0_EL1_op2 4 + +#define ICC_PPI_CPENDR1_EL1 MRS_REG_ALT_NAME(ICC_PPI_CPENDR1_EL1) +#define ICC_PPI_CPENDR1_EL1_op0 3 +#define ICC_PPI_CPENDR1_EL1_op1 0 +#define ICC_PPI_CPENDR1_EL1_CRn 12 +#define ICC_PPI_CPENDR1_EL1_CRm 13 +#define ICC_PPI_CPENDR1_EL1_op2 5 + +/* + * ICC_PPI_ENABLER_EL1 + * Interrupt Controller Physical PPI Enable Registers + */ +#define ICC_PPI_ENABLER0_EL1 MRS_REG_ALT_NAME(ICC_PPI_ENABLER0_EL1) +#define ICC_PPI_ENABLER0_EL1_op0 3 +#define ICC_PPI_ENABLER0_EL1_op1 0 +#define ICC_PPI_ENABLER0_EL1_CRn 12 +#define ICC_PPI_ENABLER0_EL1_CRm 10 +#define ICC_PPI_ENABLER0_EL1_op2 6 + +#define ICC_PPI_ENABLER1_EL1 MRS_REG_ALT_NAME(ICC_PPI_ENABLER1_EL1) +#define ICC_PPI_ENABLER1_EL1_op0 3 +#define ICC_PPI_ENABLER1_EL1_op1 0 +#define ICC_PPI_ENABLER1_EL1_CRn 12 +#define ICC_PPI_ENABLER1_EL1_CRm 10 +#define ICC_PPI_ENABLER1_EL1_op2 7 + +#define ICC_PPI_ENABLER_MASK(x) (UL(1) << ((x) & 0x3f)) +#define ICC_PPI_ENABLER_DIS(x) (UL(0) << ((x) & 0x3f)) +#define ICC_PPI_ENABLER_EN(x) (UL(1) << ((x) & 0x3f)) +#define ICC_PPI_ENABLER_NONE UL(0) + +/* + * ICC_PPI_HMR_EL1 + * Interrupt Controller Physical PPI Handling mode Register + */ +#define ICC_PPI_HMR0_EL1 MRS_REG_ALT_NAME(ICC_PPI_HMR0_EL1) +#define ICC_PPI_HMR0_EL1_op0 3 +#define ICC_PPI_HMR0_EL1_op1 0 +#define ICC_PPI_HMR0_EL1_CRn 12 +#define ICC_PPI_HMR0_EL1_CRm 10 +#define ICC_PPI_HMR0_EL1_op2 0 + +#define ICC_PPI_HMR1_EL1 MRS_REG_ALT_NAME(ICC_PPI_HMR1_EL1) +#define ICC_PPI_HMR1_EL1_op0 3 +#define ICC_PPI_HMR1_EL1_op1 0 +#define ICC_PPI_HMR1_EL1_CRn 12 +#define ICC_PPI_HMR1_EL1_CRm 10 +#define ICC_PPI_HMR1_EL1_op2 1 + +#define ICC_PPI_HMR_MASK(x) (UL(1) << ((x) & 0x3f)) +#define ICC_PPI_HMR_EDGE(x) (UL(0) << ((x) & 0x3f)) +#define ICC_PPI_HMR_LEVEL(x) (UL(1) << ((x) & 0x3f)) + +/* + * ICC_PPI_PRIORITYR_EL1 + * Interrupt Controller Physical PPI Priority Registers + */ +#define ICC_PPI_PRIORITYR0_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 0) +#define ICC_PPI_PRIORITYR1_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 1) +#define ICC_PPI_PRIORITYR2_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 2) +#define ICC_PPI_PRIORITYR3_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 3) +#define ICC_PPI_PRIORITYR4_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 4) +#define ICC_PPI_PRIORITYR5_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 5) +#define ICC_PPI_PRIORITYR6_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 6) +#define ICC_PPI_PRIORITYR7_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 14, 7) +#define ICC_PPI_PRIORITYR8_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 0) +#define ICC_PPI_PRIORITYR9_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 1) +#define ICC_PPI_PRIORITYR10_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 2) +#define ICC_PPI_PRIORITYR11_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 3) +#define ICC_PPI_PRIORITYR12_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 4) +#define ICC_PPI_PRIORITYR13_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 5) +#define ICC_PPI_PRIORITYR14_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 6) +#define ICC_PPI_PRIORITYR15_EL1 __MRS_REG_ALT_NAME(3, 0, 12, 15, 7) +#define ICC_PPI_PRIORITYR_PRIORITY_SHIFT(n) ((n) * 8) +#define ICC_PPI_PRIORITYR_PRIORITY(n, x) \ + ((uint64_t)(x) << ICC_PPI_PRIORITYR_PRIORITY_SHIFT(n)) +#define ICC_PPI_PRIORITYR_PRIORITY_ALL(x) \ + (ICC_PPI_PRIORITYR_PRIORITY(0, x) | ICC_PPI_PRIORITYR_PRIORITY(1, x) | \ + ICC_PPI_PRIORITYR_PRIORITY(2, x) | ICC_PPI_PRIORITYR_PRIORITY(3, x) | \ + ICC_PPI_PRIORITYR_PRIORITY(4, x) | ICC_PPI_PRIORITYR_PRIORITY(5, x) | \ + ICC_PPI_PRIORITYR_PRIORITY(6, x) | ICC_PPI_PRIORITYR_PRIORITY(7, x)) + +/* + * ICC_PPI_SACTIVER_EL1 + * Interrupt Controller Physical PPI Set Active Register + */ +#define ICC_PPI_SACTIVER0_EL1 MRS_REG_ALT_NAME(ICC_PPI_SACTIVER0_EL1) +#define ICC_PPI_SACTIVER0_EL1_op0 3 +#define ICC_PPI_SACTIVER0_EL1_op1 0 +#define ICC_PPI_SACTIVER0_EL1_CRn 12 +#define ICC_PPI_SACTIVER0_EL1_CRm 13 +#define ICC_PPI_SACTIVER0_EL1_op2 2 + +#define ICC_PPI_SACTIVER1_EL1 MRS_REG_ALT_NAME(ICC_PPI_SACTIVER1_EL1) +#define ICC_PPI_SACTIVER1_EL1_op0 3 +#define ICC_PPI_SACTIVER1_EL1_op1 0 +#define ICC_PPI_SACTIVER1_EL1_CRn 12 +#define ICC_PPI_SACTIVER1_EL1_CRm 13 +#define ICC_PPI_SACTIVER1_EL1_op2 3 + +/* + * ICC_PPI_SPENDR_EL1 + * Interrupt Controller Physical PPI Set Pending State Registers + */ +#define ICC_PPI_SPENDR0_EL1 MRS_REG_ALT_NAME(ICC_PPI_SPENDR0_EL1) +#define ICC_PPI_SPENDR0_EL1_op0 3 +#define ICC_PPI_SPENDR0_EL1_op1 0 +#define ICC_PPI_SPENDR0_EL1_CRn 12 +#define ICC_PPI_SPENDR0_EL1_CRm 13 +#define ICC_PPI_SPENDR0_EL1_op2 6 + +#define ICC_PPI_SPENDR1_EL1 MRS_REG_ALT_NAME(ICC_PPI_SPENDR1_EL1) +#define ICC_PPI_SPENDR1_EL1_op0 3 +#define ICC_PPI_SPENDR1_EL1_op1 0 +#define ICC_PPI_SPENDR1_EL1_CRn 12 +#define ICC_PPI_SPENDR1_EL1_CRm 13 +#define ICC_PPI_SPENDR1_EL1_op2 7 + +/* + * IRS Config Frame registers + */ +#define IRS_IDR0 0x0000 +#define IRS_IDR1 0x0004 +#define IRS_IDR2 0x0008 +#define IRS_IDR2_ISTMD_SZ_SHIFT 15 +#define IRS_IDR2_ISTMD_SZ_MASK (0x1ful << IRS_IDR2_ISTMD_SZ_SHIFT) +#define IRS_IRD2_ISTMD (0x1u << 14) +#define IRS_IDR2_IST_L2SZ_64K (0x1u << 13) +#define IRS_IDR2_IST_L2SZ_16K (0x1u << 12) +#define IRS_IDR2_IST_L2SZ_4K (0x1u << 11) +#define IRS_IDR2_IST_LEVELS (0x1u << 10) +#define IRS_IDR2_MIN_LPI_ID_BITS_SHIFT 6 +#define IRS_IDR2_MIN_LPI_ID_BITS_MASK (0xful << IRS_IDR2_MIN_LPI_ID_BITS_SHIFT) +#define IRS_IDR2_MIN_LPI_ID_BITS(x) \ + (((x) & IRS_IDR2_MIN_LPI_ID_BITS_MASK) >> IRS_IDR2_MIN_LPI_ID_BITS_SHIFT) +#define IRS_IDR2_LPI (0x1ul << 5) +#define IRS_IDR2_ID_BITS_SHIFT 0 +#define IRS_IDR2_ID_BITS_MASK (0x1ful << IRS_IDR2_ID_BITS_SHIFT) +#define IRS_IDR2_ID_BITS(x) \ + (((x) & IRS_IDR2_ID_BITS_MASK) >> IRS_IDR2_ID_BITS_SHIFT) +#define IRS_IDR3 0x000c +#define IRS_IDR4 0x0010 +#define IRS_IDR5 0x0014 +#define IRS_IDR5_SPI_RANGE 0x01ffffff +#define IRS_IDR6 0x0018 +#define IRS_IDR6_SPI_IRS_RANGE 0x00ffffff +#define IRS_IDR7 0x001c +#define IRS_IDR7_SPI_BASE 0x00ffffff +#define IRS_IIDR 0x0040 +#define IRS_AIDR 0x0044 +#define IRS_CR0 0x0080 +#define IRS_CR0_IDLE (0x1u << 1) +#define IRS_CR0_IRSEN (0x1u << 0) +#define IRS_CR1 0x0084 +#define IRS_CR1_VPED_NO_WA (0x0u << 15) +#define IRS_CR1_VPED_WA (0x1u << 15) +#define IRS_CR1_VPED_NO_RA (0x0u << 14) +#define IRS_CR1_VPED_RA (0x1u << 14) +#define IRS_CR1_VMD_NO_WA (0x0u << 13) +#define IRS_CR1_VMD_WA (0x1u << 13) +#define IRS_CR1_VMD_NO_RA (0x0u << 12) +#define IRS_CR1_VMD_RA (0x1u << 12) +#define IRS_CR1_VPET_NO_WA (0x0u << 11) +#define IRS_CR1_VPET_WA (0x1u << 11) +#define IRS_CR1_VPET_NO_RA (0x0u << 10) +#define IRS_CR1_VPET_RA (0x1u << 10) +#define IRS_CR1_VMT_NO_WA (0x0u << 9) +#define IRS_CR1_VMT_WA (0x1u << 9) +#define IRS_CR1_VMT_NO_RA (0x0u << 8) +#define IRS_CR1_VMT_RA (0x1u << 8) +#define IRS_CR1_IST_NO_WA (0x0u << 7) +#define IRS_CR1_IST_WA (0x1u << 7) +#define IRS_CR1_IST_NO_RA (0x0u << 6) +#define IRS_CR1_IST_RA (0x1u << 6) +#define IRS_CR1_IC_SHIFT 4 +#define IRS_CR1_IC_MASK (0x3u << IRS_CR1_IC_SHIFT) +#define IRS_CR1_IC_NC (0x0u << IRS_CR1_IC_SHIFT) +#define IRS_CR1_IC_WB (0x1u << IRS_CR1_IC_SHIFT) +#define IRS_CR1_IC_WT (0x2u << IRS_CR1_IC_SHIFT) +#define IRS_CR1_OC_SHIFT 2 +#define IRS_CR1_OC_MASK (0x3u << IRS_CR1_OC_SHIFT) +#define IRS_CR1_OC_MASK (0x3u << IRS_CR1_OC_SHIFT) +#define IRS_CR1_OC_NC (0x0u << IRS_CR1_OC_SHIFT) +#define IRS_CR1_OC_WB (0x1u << IRS_CR1_OC_SHIFT) +#define IRS_CR1_OC_WT (0x2u << IRS_CR1_OC_SHIFT) +#define IRS_CR1_SH_SHIFT 0 +#define IRS_CR1_SH_MASK (0x3u << IRS_CR1_SH_SHIFT) +#define IRS_CR1_SH_MASK (0x3u << IRS_CR1_SH_SHIFT) +#define IRS_CR1_SH_NS (0x0u << IRS_CR1_SH_SHIFT) +#define IRS_CR1_SH_OS (0x2u << IRS_CR1_SH_SHIFT) +#define IRS_CR1_SH_IS (0x3u << IRS_CR1_SH_SHIFT) +#define IRS_SYNCR 0x00c0 +#define IRS_SYNC_STATUSR 0x00c4 +#define IRS_SPI_VMR 0x0100 +#define IRS_SPI_SELR 0x0108 +#define IRS_SPI_DOMAINR 0x010c +#define IRS_SPI_RESAMPLER 0x0110 +#define IRS_SPI_CFGR 0x0114 +#define IRS_SPI_CFGR_TM_EDGE (0x0u << 0) +#define IRS_SPI_CFGR_TM_LEVEL (0x1u << 0) +#define IRS_SPI_STATUSR 0x0118 +#define IRS_SPI_STATUSR_V (0x1u << 1) +#define IRS_SPI_STATUSR_IDLE (0x1u << 0) +#define IRS_PE_SELR 0x0140 +#define IRS_PE_STATUSR 0x0144 +#define IRS_PE_CR0 0x0148 +#define IRS_IST_BASER 0x0180 +#define IRS_IST_BASER_ADDR_LIMIT 0x0100000000000000ul +#define IRS_IST_BASER_ADDR_MASK 0x00ffffffffffffc0ul +#define IRS_IST_BASER_VALID (0x1ul << 0) +#define IRS_IST_CFGR 0x0190 +#define IRS_IST_CFGR_STRUCTURE_LINEAR (0x0 << 16) +#define IRS_IST_CFGR_STRUCTURE_2LVL (0x1 << 16) +#define IRS_IST_CFGR_ISTSZ_SHIFT 7 +#define IRS_IST_CFGR_ISTSZ_MASK (0x3 << IRS_IST_CFGR_ISTSZ_SHIFT) +#define IRS_IST_CFGR_ISTSZ_VAL(x) \ + (((x) & IRS_IST_CFGR_ISTSZ_MASK) >> IRS_IST_CFGR_ISTSZ_SHIFT) +#define IRS_IST_CFGR_ISTSZ_4_VAL 0x0 +#define IRS_IST_CFGR_ISTSZ_4 (0x0 << IRS_IST_CFGR_ISTSZ_SHIFT) +#define IRS_IST_CFGR_ISTSZ_8_VAL 0x1 +#define IRS_IST_CFGR_ISTSZ_8 (0x1 << IRS_IST_CFGR_ISTSZ_SHIFT) +#define IRS_IST_CFGR_ISTSZ_16_VAL 0x2 +#define IRS_IST_CFGR_ISTSZ_16 (0x2 << IRS_IST_CFGR_ISTSZ_SHIFT) +#define IRS_IST_CFGR_L2SZ_SHIFT 5 +#define IRS_IST_CFGR_L2SZ_MASK (0x3 << IRS_IST_CFGR_L2SZ_SHIFT) +#define IRS_IST_CFGR_L2SZ_VAL(x) \ + (((x) & IRS_IST_CFGR_L2SZ_MASK) >> IRS_IST_CFGR_L2SZ_SHIFT) +#define IRS_IST_CFGR_L2SZ_4K_VAL 0x0 +#define IRS_IST_CFGR_L2SZ_4K (0x0 << IRS_IST_CFGR_L2SZ_SHIFT) +#define IRS_IST_CFGR_L2SZ_16K_VAL 0x1 +#define IRS_IST_CFGR_L2SZ_16K (0x1 << IRS_IST_CFGR_L2SZ_SHIFT) +#define IRS_IST_CFGR_L2SZ_64K_VAL 0x2 +#define IRS_IST_CFGR_L2SZ_64K (0x2 << IRS_IST_CFGR_L2SZ_SHIFT) +#define IRS_IST_CFGR_LPI_ID_BITS_SHIFT 0 +#define IRS_IST_CFGR_LPI_ID_BITS_MASK (0x1f << IRS_IST_CFGR_LPI_ID_BITS_SHIFT) +#define IRS_IST_CFGR_LPI_ID_BITS_VAL(x) \ + (((x) & IRS_IST_CFGR_LPI_ID_BITS_MASK) >> IRS_IST_CFGR_LPI_ID_BITS_SHIFT) +#define IRS_IST_CFGR_ +#define IRS_IST_STATUSR 0x0194 +#define IRS_IST_STATUSR_IDLE (0x1u << 0) +#define IRS_MAP_L2_ISTR 0x01c0 +#define IRS_VMT_BASER 0x0200 +#define IRS_VMT_CFGR 0x0210 +#define IRS_VMT_STATUSR 0x0214 +#define IRS_VPE_SELR 0x0240 +#define IRS_VPE_DBR 0x0248 +#define IRS_VPE_HPPIR 0x0250 +#define IRS_VPE_CR0 0x0258 +#define IRS_VPE_STATUSR 0x025c +#define IRS_VM_DBR 0x0280 +#define IRS_VM_SELR 0x0288 +#define IRS_VM_STATUSR 0x028c +#define IRS_VMAP_L2_VMTR 0x02c0 +#define IRS_VMAP_VMR 0x02c8 +#define IRS_VMAP_VISTR 0x02d0 +#define IRS_VMAP_L2_VISTR 0x02d8 +#define IRS_VMAP_VPER 0x02e0 +#define IRS_SAVE_VMR 0x0300 +#define IRS_SAVE_VM_STATUSR 0x0308 +#define IRS_MEC_IDR 0x0340 +#define IRS_MEC_MECID_R 0x0344 +#define IRS_MPAM_IDR 0x0380 +#define IRS_MPAM_PARTID_R 0x0384 +#define IRS_SWERR_STATUSR 0x03c0 +#define IRS_SWERR_SYNDROMER0 0x03c8 +#define IRS_SWERR_SYNDROMER1 0x03d0 + +/* + * IRS Data Structures + */ + +/* L1_ISTE - Level 1 interrupt state table entry */ +#define L1_ISTE_SIZE 8 +#define L1_ISTE_L2_ADDR 0xfffffffffff000ul +#define L1_ISTE_L2_ADDR_N(l2sz) (11 + 2 * (l2sz)) +#define L1_ISTE_VALID (0x1ul << 0) + +/* L2_ISTE - Level 2 interrupt state table entry */ +#define ITSE_MAX_ADDR 0x00ffffffffffffff +#define ITSE_ALIGN 8 + +#define L2_ISTE_IAFFID_SHIFT 16 +#define L2_ISTE_IAFFID_MASK (0xffffu << L2_ISTE_IAFFID_SHIFT) +#define L2_ISTE_Priority_SHIFT 11 +#define L2_ISTE_Priority (0x1fu << L2_ISTE_Priority_SHIFT) +#define L2_ISTE_HWU_SHIFT 9 +#define L2_ISTE_HWU (0x3u << L2_ISTE_HWU_SHIFT) +#define L2_ISTE_IRM (0x1u << 4) +#define L2_ISTE_Enable (0x1u << 3) +#define L2_ISTE_HM (0x1u << 2) +#define L2_ISTE_Active (0x1u << 1) +#define L2_ISTE_Pending (0x1u << 0) + +/* + * Misc + */ + +/* + * RXBQTT: In the GIC prioritization scheme, lower numbers have higher priority. + */ +#define GICV5_PRI_HIGHEST 0x0 +#define GICV5_PRI_LOWEST 0x1f + +#endif diff --git a/sys/arm64/arm64/gicv5var.h b/sys/arm64/arm64/gicv5var.h new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gicv5var.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 Arm Ltd + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _ARM64_GICV5VAR_H_ +#define _ARM64_GICV5VAR_H_ + +DECLARE_CLASS(gicv5_driver); + +struct gicv5_irqsrc; +struct gicv5_irs_irqspec; + +enum gicv5_irq_space { + GICv5_INVALID, + /* GICv5 interrupt types so they are the same values as in the spec */ + GICv5_PPI, + GICv5_LPI, + GICv5_SPI, +}; + +/* + * Shared between the IRS and ITS + */ +struct gicv5_base_irqsrc { + struct intr_irqsrc gbi_isrc; + enum gicv5_irq_space gbi_space; + uint32_t gbi_irq; +}; + +struct gicv5_irs; + +struct gicv5_softc { + device_t gic_dev; + struct intr_pic *gic_pic; + + struct gicv5_irqsrc *gic_ppi_irqs; + struct gicv5_irqsrc *gic_irs_irqs; + struct gicv5_irqsrc *gic_ipi_irqs; + struct gicv5_irs **gic_irs; + + device_t *gic_children; + + u_int gic_bus; + bool gic_coherent; + u_int gic_nirs; + u_int gic_spi_count; + u_int gic_nchildren; + /* Number of LPIs, including IPIs */ + u_int gic_nlpis; +}; + +struct gicv5_devinfo { + struct resource_list di_rl; + struct gicv5_irs *di_irs; +}; + +#define GICV5_IVAR_LPI_START 5000 + +__BUS_ACCESSOR(gicv5, lpi_start, GICV5, LPI_START, u_int); + +void gicv5_attach(device_t); +bool gicv5_add_child(device_t, struct gicv5_devinfo *); +int gicv5_intr(void *); + +void gicv5_irs_init(device_t, u_int, cpuset_t *); +void gicv5_irs_alloc_lpi(device_t, device_t, u_int); + +#endif /* _ARM64_GICV5VAR_H_ */ diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -57,6 +57,8 @@ arm64/arm64/gic_v3.c standard arm64/arm64/gic_v3_acpi.c optional acpi arm64/arm64/gic_v3_fdt.c optional fdt +arm64/arm64/gicv5.c standard +arm64/arm64/gicv5_fdt.c standard arm64/arm64/hyp_stub.S standard arm64/arm64/identcpu.c standard arm64/arm64/kexec_support.c standard