From 79d7be802edee0187df2f775d8f02b54bc49cd4a Mon Sep 17 00:00:00 2001 From: Lunaixsky Date: Wed, 9 Oct 2024 02:02:47 +0100 Subject: [PATCH] add support to MSI based on either SPI or LPI. * LPI and SPI based MSI is supported through arm gic. * re-organise the functions definition according to logical relationship. --- .../arch/aarch64/includes/asm/aa64_gic.h | 5 + lunaix-os/arch/aarch64/includes/asm/soc/gic.h | 11 +- lunaix-os/arch/aarch64/soc/gic.c | 258 +++++++++++------- .../arch/generic/includes/asm-generic/isrm.h | 2 + lunaix-os/includes/lunaix/ds/bitmap.h | 44 ++- 5 files changed, 218 insertions(+), 102 deletions(-) diff --git a/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h b/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h index ea9f691..cf787e9 100644 --- a/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h +++ b/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h @@ -45,6 +45,7 @@ typedef unsigned long gicreg64_t; #define GICD_CTLR REG_INDEX(0x0000) #define GICD_TYPER REG_INDEX(0x0004) #define GICD_IIDR REG_INDEX(0x0008) +#define GICD_SETSPI_NSR REG_INDEX(0x0040) #define GICD_IGROUPRn REG_INDEX(0x0080) #define GICD_ISENABLER REG_INDEX(0x0100) @@ -58,6 +59,7 @@ typedef unsigned long gicreg64_t; #define GICR_TYPER REG_INDEX(0x0008) #define GICR_PROPBASER REG_INDEX(0x0070) #define GICR_PENDBASER REG_INDEX(0x0078) +#define GICR_SETLPIR REG_INDEX(0x0040) #define GICD_CTLR_G1SEN BITFLAG(2) #define GICD_CTLR_G1NSEN BITFLAG(1) @@ -82,4 +84,7 @@ typedef unsigned long gicreg64_t; #define GICR_PROPBASER_IDbits\ BITFIELD(4, 0) +#define GICR_CTLR_RWP BITFLAG(31) +#define GICR_CTLR_EnLPI BITFLAG(0) + #endif /* __LUNAIX_AA64_GIC_H */ diff --git a/lunaix-os/arch/aarch64/includes/asm/soc/gic.h b/lunaix-os/arch/aarch64/includes/asm/soc/gic.h index 35b9dc5..cabcfc4 100644 --- a/lunaix-os/arch/aarch64/includes/asm/soc/gic.h +++ b/lunaix-os/arch/aarch64/includes/asm/soc/gic.h @@ -4,6 +4,9 @@ #include #include #include + +#include + #include #include @@ -182,10 +185,8 @@ struct arm_gic } mmrs; struct { - union { - ptr_t prop; - lpi_entry_t* property; - }; + ptr_t prop_pa; + lpi_entry_t* prop_table; ptr_t pend; BITMAP(gic_bmp) pendings; @@ -199,6 +200,8 @@ struct arm_gic struct gic_idomain* spi; struct gic_idomain* espi; } idomain; + + struct dt_node* gic_node; }; #endif /* __LUNAIX_GIC_H */ diff --git a/lunaix-os/arch/aarch64/soc/gic.c b/lunaix-os/arch/aarch64/soc/gic.c index ce666ea..ad24686 100644 --- a/lunaix-os/arch/aarch64/soc/gic.c +++ b/lunaix-os/arch/aarch64/soc/gic.c @@ -102,11 +102,15 @@ gic_create_from_dt() // ignore vcpu_if, as we dont do any EL2 stuff __create_its(gic_node); + + gic.gic_node = gic_node; } /* ++++++ GIC dirver ++++++ */ +/* ****** Interrupt Management ****** */ + static void __config_interrupt(struct arm_gic* gic, struct gic_distributor* dist, struct gic_interrupt* ent) @@ -121,7 +125,7 @@ __config_interrupt(struct arm_gic* gic, struct gic_distributor* dist, entry |= LPI_EN; - gic->lpi_tables.property[intid_rel] = entry; + gic->lpi_tables.prop_table[intid_rel] = entry; // clear any pending when we (re-)configuring bitmap_set(gic_bmp, &gic->lpi_tables.pendings, intid_rel, false); @@ -157,7 +161,7 @@ __undone_interrupt(struct arm_gic* gic, struct gic_distributor* dist, intid_rel = ent->intid - ent->domain->base; if (ent->config.class == GIC_LPI) { - gic->lpi_tables.property[intid_rel] = 0; + gic->lpi_tables.prop_table[intid_rel] = 0; // clear any pending when we (re-)configuring bitmap_set(gic_bmp, &gic->lpi_tables.pendings, intid_rel, false); @@ -172,57 +176,54 @@ __undone_interrupt(struct arm_gic* gic, struct gic_distributor* dist, } } -static struct gic_idomain* -__idomain(int nr_ints, unsigned int base, bool extended) + +static struct gic_interrupt* +__find_interrupt_record(unsigned int intid) { - struct gic_idomain* rec; + struct gic_idomain* domain; - rec = valloc(sizeof(*rec)); + domain = __deduce_domain(intid); - bitmap_init(gic_bmp, &rec->ivmap, nr_ints); - hashtable_init(rec->recs); + if (!domain) { + return NULL; + } - rec->base = base; - rec->extended = extended; + struct gic_interrupt *pos, *n; - return rec; + hashtable_hash_foreach(domain->recs, intid, pos, n, node) + { + if (pos->intid == intid) { + return pos; + } + } + + return NULL; } -static inline void -__init_distributor(struct gic_distributor* d, - gicreg_t* base, unsigned int nr_ints) +static inline struct gic_interrupt* +__register_interrupt(struct gic_idomain* domain, + unsigned int intid, struct gic_int_param* param) { - bitmap_init_ptr(gic_bmp, - &d->group, nr_ints, gic_regptr(base, GICD_IGROUPRn)); + struct gic_interrupt* interrupt; - bitmap_init_ptr(gic_bmp, - &d->grpmod, nr_ints, gic_regptr(base, GICD_IGRPMODRn)); + interrupt = valloc(sizeof(*interrupt)); + interrupt->config = (struct gic_intcfg) { + .class = param->class, + .trigger = param->trigger, + .group = param->group, + .as_nmi = param->as_nmi + }; - bitmap_init_ptr(gic_bmp, - &d->enable, nr_ints, gic_regptr(base, GICD_ISENABLER)); - - bitmap_init_ptr(gic_bmp, - &d->disable, nr_ints, gic_regptr(base, GICD_ICENABLER)); + interrupt->intid = intid; + interrupt->domain = domain; - bitmap_init_ptr(gic_bmp, - &d->icfg, nr_ints * 2, gic_regptr(base, GICD_ICFGR)); - - bitmap_init_ptr(gic_bmp, - &d->nmi, nr_ints, gic_regptr(base, GICD_INMIR)); -} + hashtable_hash_in(domain->recs, &interrupt->node, intid); -static inline struct leaflet* -__alloc_lpi_table(size_t table_sz) -{ - unsigned int val; - struct leaflet* tab; + return interrupt; +} - val = page_aligned(table_sz); - tab = alloc_leaflet(count_order(leaf_count(val))); - leaflet_wipe(tab); - return leaflet_addr(tab); -} +/* ****** Interrupt Domain Management ****** */ static struct gic_idomain* __deduce_domain(unsigned int intid) @@ -254,49 +255,73 @@ __deduce_domain(unsigned int intid) return NULL; } -static struct gic_interrupt* -__find_interrupt_record(unsigned int intid) +static struct gic_idomain* +__idomain(int nr_ints, unsigned int base, bool extended) { - struct gic_idomain* domain; + struct gic_idomain* rec; - domain = __deduce_domain(intid); + rec = valloc(sizeof(*rec)); - if (!domain) { - return NULL; - } + bitmap_init(gic_bmp, &rec->ivmap, nr_ints); + hashtable_init(rec->recs); - struct gic_interrupt *pos, *n; + rec->base = base; + rec->extended = extended; - hashtable_hash_foreach(domain->recs, intid, pos, n, node) - { - if (pos->intid == intid) { - return pos; - } - } + return rec; +} - return NULL; + +/* ****** Distributor-Related Management ****** */ + +static inline void +__init_distributor(struct gic_distributor* d, + gicreg_t* base, unsigned int nr_ints) +{ + bitmap_init_ptr(gic_bmp, + &d->group, nr_ints, gic_regptr(base, GICD_IGROUPRn)); + + bitmap_init_ptr(gic_bmp, + &d->grpmod, nr_ints, gic_regptr(base, GICD_IGRPMODRn)); + + bitmap_init_ptr(gic_bmp, + &d->enable, nr_ints, gic_regptr(base, GICD_ISENABLER)); + + bitmap_init_ptr(gic_bmp, + &d->disable, nr_ints, gic_regptr(base, GICD_ICENABLER)); + + bitmap_init_ptr(gic_bmp, + &d->icfg, nr_ints * 2, gic_regptr(base, GICD_ICFGR)); + + bitmap_init_ptr(gic_bmp, + &d->nmi, nr_ints, gic_regptr(base, GICD_INMIR)); } -static inline struct gic_interrupt* -__register_interrupt(struct gic_idomain* domain, - unsigned int intid, struct gic_int_param* param) +static inline struct leaflet* +__alloc_lpi_table(size_t table_sz) { - struct gic_interrupt* interrupt; + unsigned int val; + struct leaflet* tab; - interrupt = valloc(sizeof(*interrupt)); - interrupt->config = (struct gic_intcfg) { - .class = param->class, - .trigger = param->trigger, - .group = param->group, - .as_nmi = param->as_nmi - }; + val = page_aligned(table_sz); + tab = alloc_leaflet(count_order(leaf_count(val))); + leaflet_wipe(tab); - interrupt->intid = intid; - interrupt->domain = domain; + return leaflet_addr(tab); +} - hashtable_hash_in(domain->recs, &interrupt->node, intid); +static inline void +__toggle_lpi_enable(struct gic_rd* rdist, bool en) +{ + gicreg_t val; - return interrupt; + if (!en) { + while ((val = rdist->base[GICR_CTLR]) & GICR_CTLR_RWP); + rdist->base[GICR_CTLR] = val & ~GICR_CTLR_EnLPI; + } + else { + rdist->base[GICR_CTLR] = val | GICR_CTLR_EnLPI; + } } static struct gic_distributor* @@ -317,6 +342,9 @@ __attached_distributor(int cpu, struct gic_interrupt* ent) return &gic.dist; } + +/* ****** GIC Components Configuration ****** */ + static void gic_configure_icc() { @@ -399,8 +427,14 @@ gic_configure_global(struct arm_gic* gic) gic->idomain.lpi = __idomain(gic->lpi_nr, INITID_LPI_BASE, false); } - gic->lpi_tables.prop = __alloc_lpi_table(gic->lpi_nr); - gic->lpi_tables.pend = __alloc_lpi_table(gic->lpi_nr / 8); + struct leaflet* tab; + + tab = __alloc_lpi_table(gic->lpi_nr); + gic->lpi_tables.prop_table = vmap(tab, KERNEL_DATA); + gic->lpi_tables.prop_pa = leaflet_addr(tab); + + tab = __alloc_lpi_table(gic->lpi_nr / 8); + gic->lpi_tables.pend = leaflet_addr(tab); bitmap_init_ptr(gic_bmp, &gic->lpi_tables.pendings, gic->lpi_nr, gic->lpi_tables.pend); @@ -435,8 +469,10 @@ gic_configure_pe(struct arm_gic* gic, struct gic_pe* pe) __init_distributor(&pe->rdist, pe->_rd->sgi_base, nr_local_ints); + __toggle_lpi_enable(pe->_rd, false); + reg = 0; - BITS_SET(reg, GICR_BASER_PAddr, gic->lpi_tables.prop); + BITS_SET(reg, GICR_BASER_PAddr, gic->lpi_tables.prop_pa); BITS_SET(reg, GICR_BASER_Share, 0b01); BITS_SET(reg, GICR_PROPBASER_IDbits, ilog2(gic->max_intid)); pe->_rd->sgi_base[GICR_PROPBASER] = reg; @@ -446,8 +482,13 @@ gic_configure_pe(struct arm_gic* gic, struct gic_pe* pe) BITS_SET(reg, GICR_BASER_PAddr, gic->lpi_tables.pend); BITS_SET(reg, GICR_BASER_Share, 0b01); pe->_rd->sgi_base[GICR_PENDBASER] = reg; + + __toggle_lpi_enable(pe->_rd, true); } + +/* ****** Interrupt Life-cycle Management ****** */ + struct gic_interrupt* aa64_isrm_ivalloc(struct gic_int_param* param, isr_cb handler) { @@ -541,7 +582,7 @@ gic_update_active() pe->iar_val = val; } -static void +static inline void gic_signal_eoi() { struct gic_pe* pe; @@ -555,6 +596,8 @@ gic_signal_eoi() set_sysreg(ICC_EOIR1_EL1, pe->iar_val); } +/* ****** Lunaix ISRM Interfacing ****** */ + void isrm_init() { @@ -602,18 +645,6 @@ isrm_ivexalloc(isr_cb handler) return intr->intid; } -int -isrm_bindirq(int irq, isr_cb irq_handler) -{ - // Not supported -} - -void -isrm_bindiv(int iv, isr_cb handler) -{ - // Not supported -} - isr_cb isrm_get(int iv) { @@ -652,26 +683,65 @@ isrm_set_payload(int iv, ptr_t payload) } void -isrm_irq_attach(int irq, int iv, cpu_t dest, u32_t flags) +isrm_notify_eoi(cpu_t id, int iv) { - // Not supported + gic_signal_eoi(); } void -isrm_notify_eoi(cpu_t id, int iv) +isrm_notify_eos(cpu_t id) { - struct gic_interrupt* active; + isrm_notify_eoi(id, 0); +} - active = gic.pes[0].active; - assert(active); +msi_vector_t +isrm_msialloc(isr_cb handler) +{ + int intid; + msi_vector_t msiv; + struct gic_int_param param; + + param = (struct gic_int_param) { + .group = GIC_G1NS, + .trigger = GIC_TRIG_EDGE + }; + + if (gic.msi_via_spi) { + param.class = GIC_SPI; + + intid = aa64_isrm_ivalloc(¶m, handler); + msiv.msi_addr = gic_regptr(gic.mmrs.dist_base, GICD_SETSPI_NSR); + goto done; + } + + if (unlikely(!gic.lpi_ready)) { + return invalid_msi_vector; + } + + if (unlikely(gic.mmrs.its)) { + // TODO + WARN("ITS-base MSI is yet unsupported."); + return invalid_msi_vector; + } + + param.class = GIC_LPI; + intid = aa64_isrm_ivalloc(¶m, handler); + msiv.msi_addr = gic_regptr(gic.pes[0]._rd->base, GICR_SETLPIR); + +done: + msiv.mapped_iv = intid; + msiv.msi_data = intid; + + return msiv; } -void -isrm_notify_eos(cpu_t id) +int +isrm_bind_dtnode(struct dt_intr_node* node) { - isrm_notify_eoi(id, 0); + // TODO } +/* ****** Device Definition & Export ****** */ static void gic_init() diff --git a/lunaix-os/arch/generic/includes/asm-generic/isrm.h b/lunaix-os/arch/generic/includes/asm-generic/isrm.h index e8117bb..8cca7b0 100644 --- a/lunaix-os/arch/generic/includes/asm-generic/isrm.h +++ b/lunaix-os/arch/generic/includes/asm-generic/isrm.h @@ -26,6 +26,8 @@ typedef struct { #define msi_addr(msiv) ((msiv).msi_addr) #define msi_data(msiv) ((msiv).msi_data) #define msi_vect(msiv) ((msiv).mapped_iv) +#define check_msiv_invalid(msiv) (msi_vect(msiv) == -1) +#define invalid_msi_vector ((msi_vector_t) { (ptr_t)-1, (reg_t)-1, -1 }); void isrm_init(); diff --git a/lunaix-os/includes/lunaix/ds/bitmap.h b/lunaix-os/includes/lunaix/ds/bitmap.h index 68d98ba..ca02083 100644 --- a/lunaix-os/includes/lunaix/ds/bitmap.h +++ b/lunaix-os/includes/lunaix/ds/bitmap.h @@ -70,7 +70,7 @@ bitmap_##name##_query(BMP_PARAM(name), unsigned long pos) assert(pos < size); \ unsigned long n = pos / (sizeof(type) * 8); \ int i = pos % (sizeof(type) * 8); \ - type at = BMP_STRCUT_MAP[n]; \ + type at = BMP_STRCUT_MAP[n]; \ type msk = 1 << select(orient == BMP_ORIENT_MSB, \ sizeof(type) * 8 - 1 - i, i ); \ return !!(at & msk); \ @@ -84,10 +84,10 @@ bitmap_##name##_set(BMP_PARAM(name), unsigned long pos, bool val) assert(pos < size); \ unsigned long n = pos / (sizeof(type) * 8); \ int i = pos % (sizeof(type) * 8); \ - type at = BMP_STRCUT_MAP[n]; \ + type at = BMP_STRCUT_MAP[n]; \ unsigned int off = select(orient == BMP_ORIENT_MSB, \ sizeof(type) * 8 - 1 - i, i ); \ - BMP_STRCUT_MAP[n] = (at & ~(1 << off)) | (!!val << off); \ + BMP_STRCUT_MAP[n] = (at & ~(1 << off)) | (!!val << off); \ } @@ -99,15 +99,49 @@ bitmap_##name##_alloc_from(BMP_PARAM(name), unsigned long start, unsigned long i, p = 0; \ int shift; \ type u; \ + \ i = start / 8 / sizeof(type); \ shift = select(orient == BMP_ORIENT_MSB, sizeof(type) * 8 - 1, 0); \ + \ while ((u = BMP_STRCUT_MAP[i]) == (type)-1) i++; \ while ((u & (type)(1U << shift)) && p++ < sizeof(type) * 8) \ select(orient == BMP_ORIENT_MSB, u <<= 1, u >>= 1); \ + \ if (p < sizeof(type) * 8) \ return false; \ + \ BMP_STRCUT_MAP[i] |= 1UL << shift; \ - *_out = (i + p); \ + *_out = (i * 8 + p); \ + \ + return true; \ +} + +#define _DEFINE_BMP_ALLOCFROM_OP(type, size, name, orient) \ +static inline bool \ +bitmap_##name##_alloc_between(BMP_PARAM(name), \ + unsigned long start, unsigned long end, \ + unsigned long* _out) \ +{ \ + unsigned long i, p = 0; \ + unsigned long i_e, p_e = 0; \ + int shift; \ + type u; \ + \ + i = start / 8 / sizeof(type); \ + i_e = end / 8 / sizeof(type); \ + p_e = end % (8 * sizeof(type)); \ + shift = select(orient == BMP_ORIENT_MSB, sizeof(type) * 8 - 1, 0); \ + \ + while (i < i_e && (u = BMP_STRCUT_MAP[i]) == (type)-1) i++; \ + while ((u & (type)(1U << shift)) && p++ < sizeof(type) * 8) \ + select(orient == BMP_ORIENT_MSB, u <<= 1, u >>= 1); \ + \ + if (i >= i_e && p > p_e) \ + return false; \ + \ + BMP_STRCUT_MAP[i] |= 1UL << shift; \ + *_out = (i * 8 + p); \ + \ return true; \ } @@ -143,5 +177,7 @@ bitmap_##name##_alloc_from(BMP_PARAM(name), unsigned long start, #define bitmap_init_ptr(bitmap, bmp, nr_bits, ptr) \ _BMP_OP_CALL(bitmap, init_with, bmp, nr_bits, ptr) +#define bitmap_alloc_within(bitmap, bmp, start, end, out) \ + _BMP_OP_CALL(bitmap, alloc_between, bmp, start, end, out) #endif /* __LUNAIX_BITMAP_H */ -- 2.27.0