From: Lunaixsky Date: Thu, 10 Oct 2024 22:32:23 +0000 (+0100) Subject: add support to GIC ITS, and the MSI service base on it X-Git-Url: https://scm.lunaixsky.com/lunaix-os.git/commitdiff_plain/6123a1b5c5c97d90224ffa0dc6706dd7af2ebb9e?ds=sidebyside add support to GIC ITS, and the MSI service base on it --- diff --git a/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h b/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h index cf787e9..a0f3532 100644 --- a/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h +++ b/lunaix-os/arch/aarch64/includes/asm/aa64_gic.h @@ -61,6 +61,11 @@ typedef unsigned long gicreg64_t; #define GICR_PENDBASER REG_INDEX(0x0078) #define GICR_SETLPIR REG_INDEX(0x0040) +#define GITS_CTLR REG_INDEX(0x0000) +#define GITS_TYPER REG_INDEX(0x0004) +#define GITS_CBASER REG_INDEX(0x0080) +#define GITS_BASER REG_INDEX(0x0100) + #define GICD_CTLR_G1SEN BITFLAG(2) #define GICD_CTLR_G1NSEN BITFLAG(1) #define GICD_CTLR_G0EN BITFLAG(0) @@ -87,4 +92,28 @@ typedef unsigned long gicreg64_t; #define GICR_CTLR_RWP BITFLAG(31) #define GICR_CTLR_EnLPI BITFLAG(0) +#define GITS_CTLR_QS BITFLAG(31) +#define GITS_CTLR_EN BITFLAG(0) + +#define GITS_TYPER_CIL BITFLAG(36) +#define GITS_TYPER_CIDbits BITFIELD(35, 32) +#define GITS_TYPER_HCC BITFIELD(31, 24) +#define GITS_TYPER_PTA BITFLAG(19) +#define GITS_TYPER_Devbits BITFIELD(17, 13) +#define GITS_TYPER_ID_bits BITFIELD(12, 8) +#define GITS_TYPER_ITTe_sz BITFIELD(7, 4) + +#define GITS_BASER_VALID BITFLAG(63) +#define GITS_BASER_ICACHE BITFIELD(61, 59) +#define GITS_BASER_OCACHE BITFIELD(55, 53) +#define GITS_BASER_PA BITFIELD(47, 12) +#define GITS_BASER_SHARE BITFIELD(11, 10) +#define GITS_BASER_SIZE BITFIELD(7, 0) + +#define GITS_BASERn_TYPE BITFIELD(58, 56) +#define GITS_BASERn_EntSz BITFIELD(52, 48) +#define GITS_BASERn_PGSZ BITFIELD(9, 8) + +#define GITS_CWRRD_OFF BITFIELD(19, 5) + #endif /* __LUNAIX_AA64_GIC_H */ diff --git a/lunaix-os/arch/aarch64/includes/asm/soc/gic.h b/lunaix-os/arch/aarch64/includes/asm/soc/gic.h index f7c5068..840700d 100644 --- a/lunaix-os/arch/aarch64/includes/asm/soc/gic.h +++ b/lunaix-os/arch/aarch64/includes/asm/soc/gic.h @@ -10,6 +10,7 @@ #include #include +// nr of cpus, must be 1 #define NR_CPU 1 #define gic_bmp PREP_BITMAP(gicreg_t, gic_intr, BMP_ORIENT_LSB) @@ -147,19 +148,82 @@ struct gic_pe }; }; -struct gic_its +struct gic_its_regs { gicreg_t base[FRAME_LEN]; // control regs gicreg_t trn_space[FRAME_LEN]; // translation space } compact align(4); -struct gic_its_v41 +struct gic_its_regs_v41 { gicreg_t base[FRAME_LEN]; // control regs gicreg_t trn_space[FRAME_LEN]; // translation space gicreg_t vsgi_space[FRAME_LEN]; // vSGI space (v4.1+) } compact align(4); +struct gic_its_cmdqeue +{ + gicreg64_t base; + gicreg_t wr_ptr; + gicreg_t rd_ptr; +} compact align(4); + +struct gic_its_cmd +{ + gicreg64_t dw[4]; +}; + +struct gic_its_devmap +{ + struct hlist_node node; + unsigned int devid; + unsigned int next_evtid; +}; + +struct gic_its +{ + struct llist_header its; + + struct { + unsigned int nr_devid; + unsigned int nr_evtid; + unsigned int nr_cid; + unsigned int sz_itte; + + bool pta; + bool ext_cids; + }; + + union { + struct gic_its_regs* reg; + struct gic_its_regs_v41* reg_v41; + ptr_t reg_ptr; + }; + + struct { + union { + struct gic_its_cmdqeue *cmd_queue; + ptr_t cmd_queue_ptr; + }; + + union { + struct gic_its_cmd *cmds; + ptr_t cmds_ptr; + }; + + unsigned int max_cmd; + }; + + union { + struct { + gicreg_t base[8]; + } *tables; + ptr_t table_ptr; + }; + + DECLARE_HASHTABLE(devmaps, 8); +}; + typedef unsigned char lpi_entry_t; struct arm_gic @@ -179,12 +243,10 @@ struct arm_gic struct { gicreg_t* dist_base; - union { - struct gic_its* its; - struct gic_its_v41* its_v41; - }; } mmrs; + struct llist_header its; + struct { ptr_t prop_pa; lpi_entry_t* prop_table; @@ -210,4 +272,17 @@ unsigned int; gic_dtprop_interpret(struct gic_int_param* param, struct dt_prop_val* val, int width); +struct gic_its* +gic_its_create(struct arm_gic* gic, ptr_t regs); + +void +gic_configure_its(struct arm_gic* gic); + +struct gic_interrupt* +gic_install_int(struct gic_int_param* param, isr_cb handler, bool alloc); + +unsigned int +gic_its_map_lpi(struct gic_its* its, + unsigned int devid, unsigned int lpid); + #endif /* __LUNAIX_GIC_H */ diff --git a/lunaix-os/arch/aarch64/soc/gic/gic.c b/lunaix-os/arch/aarch64/soc/gic/gic.c index 1c1dd07..59905e8 100644 --- a/lunaix-os/arch/aarch64/soc/gic/gic.c +++ b/lunaix-os/arch/aarch64/soc/gic/gic.c @@ -413,8 +413,8 @@ gic_configure_pe(struct arm_gic* gic, struct gic_pe* pe) /* ****** Interrupt Life-cycle Management ****** */ -static struct gic_interrupt* -__gic_install_int(struct gic_int_param* param, isr_cb handler, bool alloc) +struct gic_interrupt* +gic_install_int(struct gic_int_param* param, isr_cb handler, bool alloc) { unsigned int iv; struct gic_idomain* domain; @@ -573,7 +573,7 @@ isrm_ivexalloc(isr_cb handler) .trigger = GIC_TRIG_EDGE, }; - intr = __gic_install_int(¶m, handler, true); + intr = gic_install_int(¶m, handler, true); return intr->intid; } @@ -642,7 +642,7 @@ isrm_msialloc(isr_cb handler) if (gic.msi_via_spi) { param.class = GIC_SPI; - intid = __gic_install_int(¶m, handler, true); + intid = gic_install_int(¶m, handler, true); msiv.msi_addr = gic_regptr(gic.mmrs.dist_base, GICD_SETSPI_NSR); goto done; } @@ -651,14 +651,14 @@ isrm_msialloc(isr_cb handler) return invalid_msi_vector; } - if (unlikely(gic.mmrs.its)) { - // TODO + if (unlikely(llist_empty(&gic.its))) { + // FIXME The MSI interface need rework WARN("ITS-base MSI is yet unsupported."); return invalid_msi_vector; } param.class = GIC_LPI; - intid = __gic_install_int(¶m, handler, true); + intid = gic_install_int(¶m, handler, true); msiv.msi_addr = gic_regptr(gic.pes[0]._rd->base, GICR_SETLPIR); done: @@ -675,7 +675,7 @@ isrm_bind_dtnode(struct dt_intr_node* node, isr_cb handler) struct gic_int_param param; struct gic_interrupt* installed; - val = resolve_interrupt(INTR_TO_DTNODE(node)); + val = dt_resolve_interrupt(INTR_TO_DTNODE(node)); if (!val) { return EINVAL; } @@ -688,7 +688,7 @@ isrm_bind_dtnode(struct dt_intr_node* node, isr_cb handler) gic_dtprop_interpret(¶m, val, 3); param.cpu_id = 0; - installed = __gic_install_int(¶m, handler, false); + installed = gic_install_int(¶m, handler, false); return installed->intid; } @@ -713,6 +713,8 @@ gic_init() { gic_configure_pe(&gic, &gic.pes[i]); } + + gic_configure_its(&gic); } static struct device_def dev_arm_gic = { diff --git a/lunaix-os/arch/aarch64/soc/gic/gic_dt.c b/lunaix-os/arch/aarch64/soc/gic/gic_dt.c index 863aa03..cce0411 100644 --- a/lunaix-os/arch/aarch64/soc/gic/gic_dt.c +++ b/lunaix-os/arch/aarch64/soc/gic/gic_dt.c @@ -1,4 +1,5 @@ #include +#include #include #include @@ -20,25 +21,15 @@ __its_predicate(struct dt_node_iter* iter, struct dt_node_base* pos) } static void -__setup_pe_rdist(struct arm_gic* gic, struct dt_prop_iter* prop) +__setup_pe_rdist(struct arm_gic* gic, struct dt_prop_iter* prop, int cpu) { ptr_t base; - size_t len, off; - int i; + size_t len; base = dtprop_reg_nextaddr(prop); len = dtprop_reg_nextlen(prop); - assert(len >= NR_CPU * FRAME_SIZE * 2); - - i = 0; - base = ioremap(base, len); - off = base; - - for (; i < NR_CPU; i++) { - gic->pes[i]._rd = (struct gic_rd*) (base + off); - off += sizeof(struct gic_rd); - } + gic->pes[cpu]._rd = (struct gic_rd*)ioremap(base, len); } static void @@ -47,25 +38,24 @@ __create_its(struct arm_gic* gic, struct dt_node* gic_node) struct dt_node* its_node; struct dt_node_iter iter; struct dt_prop_iter prop; + struct gic_its* its; ptr_t its_base; size_t its_size; dt_begin_find(&iter, gic_node, __its_predicate, NULL); - if (!dt_find_next(&iter, (struct dt_node_base**)&its_node)) { - return; - } - - dt_end_find(&iter); - - dt_decode_reg(&prop, its_node, reg); - - its_base = dtprop_reg_nextaddr(&prop); - its_size = dtprop_reg_nextlen(&prop); + while (dt_find_next(&iter, (struct dt_node_base**)&its_node)) + { + dt_decode_reg(&prop, its_node, reg); - assert(its_size >= sizeof(struct gic_its)); + its_base = dtprop_reg_nextaddr(&prop); + its_size = dtprop_reg_nextlen(&prop); - gic->mmrs.its = (struct gic_its*)ioremap(its_base, its_size); + its = gic_its_create(gic, ioremap(its_base, its_size)); + dt_bind_object(&its_node->base, its); + } + + dt_end_find(&iter); } void @@ -91,11 +81,11 @@ gic_create_from_dt(struct arm_gic* gic) sz = dtprop_reg_nextlen(&prop); gic->mmrs.dist_base = (gicreg_t*)ioremap(ptr, sz); - __setup_pe_rdist(gic, &prop); - - // ignore cpu_if, as we use sysreg to access them - dtprop_next_n(&prop, 2); + for (int i = 0; i < NR_CPU; i++) { + __setup_pe_rdist(gic, &prop, i); + } + // ignore cpu_if, as we use sysreg to access them // ignore vcpu_if, as we dont do any EL2 stuff __create_its(gic, gic_node); @@ -107,10 +97,10 @@ unsigned int; gic_dtprop_interpret(struct gic_int_param* param, struct dt_prop_val* val, int width) { - struct dt_prop_iter* iter; + struct dt_prop_iter iter; unsigned int v; - dt_decode(&iter, NULL, val, 1); + dt_decode_simple(&iter, val); v = dtprop_u32_at(&iter, 0); switch (v) diff --git a/lunaix-os/arch/aarch64/soc/gic/gic_its.c b/lunaix-os/arch/aarch64/soc/gic/gic_its.c new file mode 100644 index 0000000..5874cab --- /dev/null +++ b/lunaix-os/arch/aarch64/soc/gic/gic_its.c @@ -0,0 +1,245 @@ +#include +#include + +#include + +struct gic_its* +gic_its_create(struct arm_gic* gic, ptr_t regs) +{ + struct gic_its* its; + + its = valloc(sizeof(*its)); + + its->reg_ptr = regs; + its->cmd_queue_ptr = gic_regptr(its->reg->base, GITS_CBASER); + its->table_ptr = gic_regptr(its->reg->base, GITS_BASER); + + llist_append(&gic->its, &its->its); + hashtable_init(its->devmaps); + + return its; +} + +ptr_t +__pack_base_reg(struct leaflet* page, int ic, int oc) +{ + ptr_t reg = 0; + + reg |= GITS_BASER_VALID; + BITS_SET(reg, GITS_BASER_ICACHE, ic); + BITS_SET(reg, GITS_BASER_OCACHE, oc); + BITS_SET(reg, GITS_BASER_PA, leaflet_addr(page)); + BITS_SET(reg, GITS_BASER_SIZE, 1); + BITS_SET(reg, GITS_BASER_SHARE, 0b01); + + return reg; +} + +static void +__configure_table_desc(struct gic_its* its, unsigned int off) +{ + gicreg_t val; + unsigned int type, sz; + + val = its->tables->base[off]; + type = BITS_GET(val, GITS_BASERn_TYPE); + sz = BITS_GET(val, GITS_BASERn_EntSz) + 1; + + switch (type) + { + case 0b001: // DeviceID + its->nr_devid = PAGE_SIZE / sz; + break; + case 0b100: // CollectionID + its->nr_cid = PAGE_SIZE / sz; + break; + default: + return; + } + + val = __pack_base_reg(alloc_leaflet(0), 0, 0); + BITS_SET(val, GITS_BASERn_TYPE, type); + BITS_SET(val, GITS_BASERn_EntSz, sz); + BITS_SET(val, GITS_BASERn_PGSZ, 0); // 4K + + its->tables->base[off] = val; +} + +static void +__configure_one_its(struct arm_gic* gic, struct gic_its* its) +{ + struct leaflet *leaflet; + struct gic_its_regs *reg; + gicreg_t val; + unsigned int field_val; + + reg = its->reg; + wait_until(!(val = reg->base[GITS_CTLR]) & GITS_CTLR_QS); + reg->base[GITS_CTLR] = val & ~GITS_CTLR_EN; + + val = reg->base[GITS_TYPER]; + + field_val = BITS_GET(val, GITS_TYPER_Devbits); + its->nr_devid = 1 << (field_val + 1); + + field_val = BITS_GET(val, GITS_TYPER_ID_bits); + its->nr_evtid = 1 << (field_val + 1); + + field_val = BITS_GET(val, GITS_TYPER_ITTe_sz); + its->nr_evtid = field_val + 1; + + if (!(val & GITS_TYPER_CIL)) { + its->nr_cid = BITS_GET(val, GITS_TYPER_HCC); + } + else { + field_val = BITS_GET(val, GITS_TYPER_CIDbits); + its->nr_cid = field_val + 1; + its->ext_cids = true; + } + + its->pta = !!(val & GITS_TYPER_PTA); + + leaflet = alloc_leaflet_pinned(0); + val = __pack_base_reg(leaflet, 0, 0); + its->cmd_queue->base = val; + its->cmd_queue->wr_ptr = 0; + its->cmds_ptr = vmap(leaflet, KERNEL_DATA); + its->max_cmd = leaflet_size(leaflet) / sizeof(struct gic_its_cmd); + + // total of 8 GITS_BASER registers + for (int i = 0; i < 8; i++) + { + __configure_table_desc(its, i); + } + + reg->base[GITS_CTLR] |= GITS_CTLR_EN; +} + +static void +__submit_itscmd(struct gic_its* its, struct gic_its_cmd* cmd) +{ + gicreg_t reg; + ptr_t wrp; + + reg = its->cmd_queue->wr_ptr; + + wrp = BITS_GET(reg, GITS_CWRRD_OFF); + + if (wrp == its->max_cmd) { + /* + it is very unlikely we will submit the commands fast + enough to satiate the entire queue. so we just roll + back to front, assume the ITS always keeping up. + */ + wrp = 0; + } + + its->cmds[wrp++] = *cmd; + + reg = BITS_SET(reg, GITS_CWRRD_OFF, wrp); + its->cmd_queue->wr_ptr = reg; +} + +static void +__build_mapc(struct gic_its *its, struct gic_its_cmd* cmd, + struct gic_rd* rd, unsigned int cid) +{ + cmd->dw[0] = 0x09; + cmd->dw[1] = 0; + cmd->dw[3] = 0; + cmd->dw[2] = (1UL << 63) | + (__ptr(rd) & ~0xffff) | + (cid & 0xffff); +} + +static void +__build_mapd(struct gic_its *its, struct gic_its_cmd* cmd, + struct leaflet *itt_page, unsigned int devid) +{ + cmd->dw[0] = ((gicreg64_t)devid << 32) | 0x08; + cmd->dw[1] = ilog2(its->nr_evtid); + cmd->dw[3] = 0; + cmd->dw[2] = (1UL << 63) | + (leaflet_addr(itt_page)); +} + +static void +__build_mapti(struct gic_its *its, struct gic_its_cmd* cmd, + unsigned int devid, unsigned int evtid, + unsigned int lpid, unsigned int cid) +{ + cmd->dw[0] = ((gicreg64_t)devid << 32) | 0x0A; + cmd->dw[1] = ((gicreg64_t)evtid << 32) | evtid; + cmd->dw[3] = 0; + cmd->dw[2] = (cid & 0xffff); +} + +static unsigned int +__alloc_evtid(struct gic_its *its, unsigned int devid) +{ + unsigned int evtid; + struct gic_its_devmap *pos, *n; + struct gic_its_cmd cmd; + + hashtable_hash_foreach(its->devmaps, devid, pos, n, node) + { + if (pos->devid == devid) { + goto found; + } + } + + pos = valloc(sizeof(*pos)); + pos->next_evtid = 0; + pos->devid = devid; + hashtable_hash_in(its->devmaps, &pos->node, devid); + + __build_mapd(its, &cmd, alloc_leaflet_pinned(0), devid); + __submit_itscmd(its, &cmd); + +found: + evtid = pos->next_evtid++; + + assert(pos->next_evtid < its->nr_evtid); + + return evtid; +} + +unsigned int +gic_its_map_lpi(struct gic_its* its, + unsigned int devid, unsigned int lpid) +{ + unsigned int evtid; + struct gic_its_cmd cmd; + + evtid = __alloc_evtid(its, devid); + + __build_mapti(its, &cmd, devid, evtid, lpid, 0); + __submit_itscmd(its, &cmd); + + return evtid; +} + +void +gic_configure_its(struct arm_gic* gic) +{ + struct gic_its *pos, *n; + struct gic_its_cmd cmd; + + llist_for_each(pos, n, &gic->its, its) + { + __configure_one_its(gic, pos); + } + + // halt the cpu for a while to let ITS warming up + wait_until_expire(true, 10000); + + // identity map every re-distributor to collection + llist_for_each(pos, n, &gic->its, its) + { + for (int i = 0; i < NR_CPU; i++) + { + __build_mapc(pos, &cmd, gic->pes[i]._rd, i); + __submit_itscmd(pos, &cmd); + } + } +} \ No newline at end of file