1 #include <lunaix/types.h>
2 #include <lunaix/device.h>
3 #include <lunaix/spike.h>
4 #include <lunaix/mm/valloc.h>
5 #include <lunaix/mm/page.h>
6 #include <lunaix/mm/mmio.h>
7 #include <lunaix/syslog.h>
9 #include <hal/devtree.h>
11 #include <asm/aa64_isrm.h>
12 #include <asm/soc/gic.h>
14 static struct arm_gic gic;
18 DEFINE_BMP_INIT_OP(gic_bmp, valloc);
20 DEFINE_BMP_QUERY_OP(gic_bmp);
22 DEFINE_BMP_SET_OP(gic_bmp);
24 DEFINE_BMP_ALLOCFROM_OP(gic_bmp);
27 /* ++++++ GIC device-tree retrieval ++++++ */
30 __setup_pe_rdist(struct dt_prop_iter* prop)
36 base = dtprop_reg_nextaddr(prop);
37 len = dtprop_reg_nextlen(prop);
39 assert(len >= NR_CPU * FRAME_SIZE * 2);
42 base = ioremap(base, len);
45 for (; i < NR_CPU; i++) {
46 gic.pes[i]._rd = (struct gic_rd*) (base + off);
47 off += sizeof(struct gic_rd);
52 __create_its(struct dt_node* gic_node)
54 struct dt_node* its_node;
55 struct dt_node_iter iter;
56 struct dt_prop_iter prop;
60 dt_begin_find(&iter, gic_node, "its");
62 if (!dt_find_next(&iter, (struct dt_node_base**)&its_node)) {
66 dt_decode_reg(&prop, its_node, reg);
68 its_base = dtprop_reg_nextaddr(&prop);
69 its_size = dtprop_reg_nextlen(&prop);
71 assert(its_size >= sizeof(struct gic_its));
73 gic.mmrs.its = (struct gic_its*)ioremap(its_base, its_size);
79 struct dt_node* gic_node;
80 struct dt_node_iter iter;
81 struct dt_prop_iter prop;
85 dt_begin_find(&iter, NULL, "interrupt-controller");
87 if (!dt_find_next(&iter, (struct dt_node_base**)&gic_node)) {
88 fail("expected /interrupt-controller node, but found none");
91 dt_decode_reg(&prop, gic_node, reg);
93 ptr = dtprop_reg_nextaddr(&prop);
94 sz = dtprop_reg_nextlen(&prop);
95 gic.mmrs.dist_base = (gicreg_t*)ioremap(ptr, sz);
97 __setup_pe_rdist(&prop);
99 // ignore cpu_if, as we use sysreg to access them
100 dtprop_next_n(&prop, 2);
102 // ignore vcpu_if, as we dont do any EL2 stuff
104 __create_its(gic_node);
106 gic.gic_node = gic_node;
110 /* ++++++ GIC dirver ++++++ */
112 /* ****** Interrupt Management ****** */
115 __config_interrupt(struct arm_gic* gic, struct gic_distributor* dist,
116 struct gic_interrupt* ent)
118 unsigned int intid_rel;
119 unsigned long trig_index;
121 intid_rel = ent->intid - ent->domain->base;
123 if (ent->config.class == GIC_LPI) {
124 lpi_entry_t entry = 0;
128 gic->lpi_tables.prop_table[intid_rel] = entry;
130 // clear any pending when we (re-)configuring
131 bitmap_set(gic_bmp, &gic->lpi_tables.pendings, intid_rel, false);
137 bitmap_set(gic_bmp, &dist->group, intid_rel, 0);
138 bitmap_set(gic_bmp, &dist->grpmod, intid_rel, 1);
140 trig_index = intid_rel * 2;
141 bitmap_set(gic_bmp, &dist->icfg, trig_index, 0);
142 if (ent->config.trigger == GIC_TRIG_EDGE) {
143 bitmap_set(gic_bmp, &dist->icfg, trig_index + 1, 1);
145 bitmap_set(gic_bmp, &dist->icfg, trig_index + 1, 0);
148 if (gic->nmi_ready) {
149 bitmap_set(gic_bmp, &dist->nmi, intid_rel, ent->config.as_nmi);
152 bitmap_set(gic_bmp, &dist->enable, intid_rel, true);
156 __undone_interrupt(struct arm_gic* gic, struct gic_distributor* dist,
157 struct gic_interrupt* ent)
159 unsigned int intid_rel;
161 intid_rel = ent->intid - ent->domain->base;
163 if (ent->config.class == GIC_LPI) {
164 gic->lpi_tables.prop_table[intid_rel] = 0;
166 // clear any pending when we (re-)configuring
167 bitmap_set(gic_bmp, &gic->lpi_tables.pendings, intid_rel, false);
172 bitmap_set(gic_bmp, &dist->disable, intid_rel, true);
174 if (gic->nmi_ready) {
175 bitmap_set(gic_bmp, &dist->nmi, intid_rel, false);
180 static struct gic_interrupt*
181 __find_interrupt_record(unsigned int intid)
183 struct gic_idomain* domain;
185 domain = __deduce_domain(intid);
191 struct gic_interrupt *pos, *n;
193 hashtable_hash_foreach(domain->recs, intid, pos, n, node)
195 if (pos->intid == intid) {
203 static inline struct gic_interrupt*
204 __register_interrupt(struct gic_idomain* domain,
205 unsigned int intid, struct gic_int_param* param)
207 struct gic_interrupt* interrupt;
209 interrupt = valloc(sizeof(*interrupt));
210 interrupt->config = (struct gic_intcfg) {
211 .class = param->class,
212 .trigger = param->trigger,
213 .group = param->group,
214 .as_nmi = param->as_nmi
217 interrupt->intid = intid;
218 interrupt->domain = domain;
220 hashtable_hash_in(domain->recs, &interrupt->node, intid);
226 /* ****** Interrupt Domain Management ****** */
228 static struct gic_idomain*
229 __deduce_domain(unsigned int intid)
231 if (intid <= INITID_SGI_END) {
232 return gic.pes[0].idomain.local_ints;
235 if (intid <= INITID_PPI_END) {
236 return gic.pes[0].idomain.local_ints;
239 if (intid <= INITID_SPI_END) {
240 return gic.idomain.spi;
243 if (INITID_ePPI_BASE <= intid && intid <= INITID_ePPI_END) {
244 return gic.pes[0].idomain.eppi;
247 if (INITID_eSPI_BASE <= intid && intid <= INITID_eSPI_END) {
248 return gic.idomain.espi;
251 if (intid >= INITID_LPI_BASE) {
252 return gic.idomain.lpi;
258 static struct gic_idomain*
259 __idomain(int nr_ints, unsigned int base, bool extended)
261 struct gic_idomain* rec;
263 rec = valloc(sizeof(*rec));
265 bitmap_init(gic_bmp, &rec->ivmap, nr_ints);
266 hashtable_init(rec->recs);
269 rec->extended = extended;
275 /* ****** Distributor-Related Management ****** */
278 __init_distributor(struct gic_distributor* d,
279 gicreg_t* base, unsigned int nr_ints)
281 bitmap_init_ptr(gic_bmp,
282 &d->group, nr_ints, gic_regptr(base, GICD_IGROUPRn));
284 bitmap_init_ptr(gic_bmp,
285 &d->grpmod, nr_ints, gic_regptr(base, GICD_IGRPMODRn));
287 bitmap_init_ptr(gic_bmp,
288 &d->enable, nr_ints, gic_regptr(base, GICD_ISENABLER));
290 bitmap_init_ptr(gic_bmp,
291 &d->disable, nr_ints, gic_regptr(base, GICD_ICENABLER));
293 bitmap_init_ptr(gic_bmp,
294 &d->icfg, nr_ints * 2, gic_regptr(base, GICD_ICFGR));
296 bitmap_init_ptr(gic_bmp,
297 &d->nmi, nr_ints, gic_regptr(base, GICD_INMIR));
300 static inline struct leaflet*
301 __alloc_lpi_table(size_t table_sz)
306 val = page_aligned(table_sz);
307 tab = alloc_leaflet(count_order(leaf_count(val)));
310 return leaflet_addr(tab);
314 __toggle_lpi_enable(struct gic_rd* rdist, bool en)
319 while ((val = rdist->base[GICR_CTLR]) & GICR_CTLR_RWP);
320 rdist->base[GICR_CTLR] = val & ~GICR_CTLR_EnLPI;
323 rdist->base[GICR_CTLR] = val | GICR_CTLR_EnLPI;
327 static struct gic_distributor*
328 __attached_distributor(int cpu, struct gic_interrupt* ent)
330 enum gic_int_type iclass;
332 iclass = ent->config.class;
334 if (iclass == GIC_PPI || iclass == GIC_SGI) {
335 return &gic.pes[cpu].rdist;
338 if (ent->domain->extended) {
346 /* ****** GIC Components Configuration ****** */
354 sysreg_flagging(ICC_SRE_EL1,
355 ICC_SRE_SRE | ICC_SRE_DFB | ICC_SRE_DIB,
360 sysreg_flagging(ICC_CTLR_EL1,
362 ICC_CTRL_EOImode | ICC_CTRL_PMHE);
364 // disable all group 0 interrupts as those are meant for EL3
366 sysreg_flagging(ICC_IGRPEN0_EL1, 0, ICC_IGRPEN_ENABLE);
368 // enable all group 1 interrupts, we'll stick with EL1_NS
370 sysreg_flagging(ICC_IGRPEN1_EL1, ICC_IGRPEN_ENABLE, 0);
374 gic_configure_global(struct arm_gic* gic)
377 unsigned int val, max_nr_spi;
379 reg = gic->mmrs.dist_base[GICD_TYPER];
381 // check if eSPI supported
382 gic->has_espi = (reg & GICD_TYPER_ESPI);
384 val = BITS_GET(reg, GICD_TYPER_nESPI);
385 gic->espi_nr = 32 * (val + 1);
389 val = BITS_GET(reg, GICD_TYPER_IDbits);
390 gic->max_intid = 1 << (val + 1) - 1;
394 val = BITS_GET(reg, GICD_TYPER_nLPI);
396 gic->lpi_nr = 1 << (val + 1);
399 gic->lpi_nr = gic->max_intid - INITID_LPI_BASE + 1;
403 // check if SPI supported
404 val = BITS_GET(reg, GICD_TYPER_nSPI);
406 max_nr_spi = 32 * (val + 1);
407 gic->spi_nr = MIN(max_nr_spi, INITID_SPEC_BASE);
408 gic->spi_nr -= INITID_SPI_BASE;
413 gic->nmi_ready = (reg & GICD_TYPER_NMI);
414 gic->msi_via_spi = (reg & GICD_TYPER_MBIS);
416 __init_distributor(&gic->dist, gic->mmrs.dist_base, gic->spi_nr);
417 __init_distributor(&gic->dist_e, gic->mmrs.dist_base, gic->espi_nr);
421 gic->idomain.spi = __idomain(gic->spi_nr, INITID_SPI_BASE, false);
424 gic->idomain.espi = __idomain(gic->espi_nr, INITID_eSPI_BASE, true);
427 gic->idomain.lpi = __idomain(gic->lpi_nr, INITID_LPI_BASE, false);
432 tab = __alloc_lpi_table(gic->lpi_nr);
433 gic->lpi_tables.prop_table = vmap(tab, KERNEL_DATA);
434 gic->lpi_tables.prop_pa = leaflet_addr(tab);
436 tab = __alloc_lpi_table(gic->lpi_nr / 8);
437 gic->lpi_tables.pend = leaflet_addr(tab);
439 bitmap_init_ptr(gic_bmp,
440 &gic->lpi_tables.pendings, gic->lpi_nr, gic->lpi_tables.pend);
444 gic_configure_pe(struct arm_gic* gic, struct gic_pe* pe)
446 unsigned int nr_local_ints;
449 reg = gic_reg64(pe->_rd, GICR_TYPER);
451 pe->affinity = BITS_GET(reg, GICR_TYPER_AffVal);
452 pe->ppi_nr = INITID_PPI_BASE;
453 switch (BITS_GET(reg, GICR_TYPER_PPInum))
456 pe->ppi_nr += 1088 - INITID_ePPI_BASE;
457 pe->eppi_ready = true;
460 pe->ppi_nr += 1120 - INITID_ePPI_BASE;
461 pe->eppi_ready = true;
465 nr_local_ints = pe->ppi_nr + INITID_PPI_BASE;
467 pe->idomain.local_ints = __idomain(32, 0, false);
468 pe->idomain.eppi = __idomain(nr_local_ints - 32, INITID_ePPI_BASE, true);
470 __init_distributor(&pe->rdist, pe->_rd->sgi_base, nr_local_ints);
472 __toggle_lpi_enable(pe->_rd, false);
475 BITS_SET(reg, GICR_BASER_PAddr, gic->lpi_tables.prop_pa);
476 BITS_SET(reg, GICR_BASER_Share, 0b01);
477 BITS_SET(reg, GICR_PROPBASER_IDbits, ilog2(gic->max_intid));
478 pe->_rd->sgi_base[GICR_PROPBASER] = reg;
481 reg |= GICR_PENDBASER_PTZ;
482 BITS_SET(reg, GICR_BASER_PAddr, gic->lpi_tables.pend);
483 BITS_SET(reg, GICR_BASER_Share, 0b01);
484 pe->_rd->sgi_base[GICR_PENDBASER] = reg;
486 __toggle_lpi_enable(pe->_rd, true);
490 /* ****** Interrupt Life-cycle Management ****** */
492 struct gic_interrupt*
493 aa64_isrm_ivalloc(struct gic_int_param* param, isr_cb handler)
496 struct gic_idomain* domain;
503 switch (param->class)
506 if (!param->ext_range) {
507 domain = gic.pes[cpu].idomain.local_ints;
510 domain = gic.pes[cpu].idomain.eppi;
515 domain = gic.pes[cpu].idomain.local_ints;
519 if (!param->ext_range) {
520 assert(gic.spi_nr > 0);
521 domain = gic.idomain.spi;
524 assert(gic.has_espi);
525 domain = gic.idomain.espi;
530 assert(gic.lpi_ready);
531 domain = gic.idomain.lpi;
535 fail("unknown interrupt class");
539 if (!bitmap_alloc(gic_bmp, &domain->ivmap, 0, &iv)) {
540 FATAL("out of usable iv for class=%d", param->class);
545 if (param->class == GIC_SPI && !param->ext_range && iv >= INITID_ePPI_BASE)
547 WARN("PPI vector=%d falls in extended range, while not requested.", iv);
548 param->ext_range = true;
551 struct gic_interrupt* ent;
552 struct gic_distributor* dist;
554 ent = __register_interrupt(domain, iv, param);
555 dist = __attached_distributor(cpu, ent);
557 __config_interrupt(&gic, dist, ent);
559 ent->handler = handler;
569 struct gic_interrupt* intr;
573 val = read_sysreg(ICC_IAR1_EL1);
574 intid = (unsigned int)val & ((1 << 24) - 1);
576 if (check_special_intid(intid)) {
580 intr = __find_interrupt_record(intid);
596 set_sysreg(ICC_EOIR1_EL1, pe->iar_val);
599 /* ****** Lunaix ISRM Interfacing ****** */
610 struct gic_interrupt* ent;
611 struct gic_distributor* dist;
613 ent = __find_interrupt_record(iv);
618 dist = __attached_distributor(0, ent);
619 __undone_interrupt(&gic, dist, ent);
621 hlist_delete(&ent->node);
626 isrm_ivosalloc(isr_cb handler)
628 return isrm_ivexalloc(handler);
632 isrm_ivexalloc(isr_cb handler)
634 struct gic_int_param param;
635 struct gic_interrupt* intr;
637 param = (struct gic_int_param) {
640 .trigger = GIC_TRIG_EDGE,
643 intr = aa64_isrm_ivalloc(¶m, handler);
651 struct gic_interrupt* intr;
653 intr = __find_interrupt_record(iv);
658 return intr->handler;
662 isrm_get_payload(const struct hart_state* state)
664 struct gic_interrupt* active;
666 active = gic.pes[0].active;
669 return active->handler;
673 isrm_set_payload(int iv, ptr_t payload)
675 struct gic_interrupt* intr;
677 intr = __find_interrupt_record(iv);
682 intr->payload = payload;
686 isrm_notify_eoi(cpu_t id, int iv)
692 isrm_notify_eos(cpu_t id)
694 isrm_notify_eoi(id, 0);
698 isrm_msialloc(isr_cb handler)
702 struct gic_int_param param;
704 param = (struct gic_int_param) {
706 .trigger = GIC_TRIG_EDGE
709 if (gic.msi_via_spi) {
710 param.class = GIC_SPI;
712 intid = aa64_isrm_ivalloc(¶m, handler);
713 msiv.msi_addr = gic_regptr(gic.mmrs.dist_base, GICD_SETSPI_NSR);
717 if (unlikely(!gic.lpi_ready)) {
718 return invalid_msi_vector;
721 if (unlikely(gic.mmrs.its)) {
723 WARN("ITS-base MSI is yet unsupported.");
724 return invalid_msi_vector;
727 param.class = GIC_LPI;
728 intid = aa64_isrm_ivalloc(¶m, handler);
729 msiv.msi_addr = gic_regptr(gic.pes[0]._rd->base, GICR_SETLPIR);
732 msiv.mapped_iv = intid;
733 msiv.msi_data = intid;
739 isrm_bind_dtnode(struct dt_intr_node* node)
744 /* ****** Device Definition & Export ****** */
749 memset(&gic, 0, sizeof(gic));
751 gic_create_from_dt();
753 // configure the system interfaces
756 // configure global distributor
757 gic_configure_global(&gic);
759 // configure per-PE local distributor (redistributor)
760 for (int i = 0; i < NR_CPU; i++)
762 gic_configure_pe(&gic, &gic.pes[i]);
766 static struct device_def dev_arm_gic = {
767 .name = "ARM Generic Interrupt Controller",
768 .class = DEVCLASS(DEVIF_SOC, DEVFN_CFG, DEV_INTC),
771 EXPORT_DEVICE(arm_gic, &dev_arm_gic, load_sysconf);