From 3ed7dcc4a8ccf443d125e7908d8293b562c68d4b Mon Sep 17 00:00:00 2001 From: Keqian Zhu Date: Fri, 10 Apr 2020 13:15:35 +0800 Subject: [PATCH] intc/gicv3: Add pre-sizing capability to GICv3 Currently GICv3 supports fixed smp_cpus CPUs, and all CPUs are present always. Now we want to pre-sizing GICv3 to support max_cpus CPUs and not all of them are present always, so some sizing codes should be concerned. GIC irqs, GICR and GICC are pre-created for all possible CPUs at start, but only smp_cpus CPUs are realize and irqs of smp_cpus CPUs are connected. Other code changes are mainly for arm_gicv3, and we do little about kvm_arm_gicv3 becasue KVM will deal with the sizing information properly. Signed-off-by: Keqian Zhu Signed-off-by: Salil Mehta --- hw/arm/virt.c | 17 +++++++++++---- hw/intc/arm_gicv3.c | 43 +++++++++++++++++++++++++------------- hw/intc/arm_gicv3_common.c | 22 +++++++++++++++++-- hw/intc/arm_gicv3_cpuif.c | 4 ++++ hw/intc/arm_gicv3_kvm.c | 28 ++++++++++++++++++++++++- include/hw/arm/virt.h | 3 ++- 6 files changed, 95 insertions(+), 22 deletions(-) diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 0af0a996a1..b1224fb1e4 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -827,14 +827,19 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) SysBusDevice *gicbusdev; const char *gictype; int type = vms->gic_version, i; + /* The max number of CPUs suppored by GIC */ + unsigned int num_cpus = ms->smp.cpus; + /* The number of CPUs present before boot */ unsigned int smp_cpus = ms->smp.cpus; uint32_t nb_redist_regions = 0; + assert(num_cpus >= smp_cpus); + gictype = (type == 3) ? gicv3_class_name() : gic_class_name(); vms->gic = qdev_new(gictype); qdev_prop_set_uint32(vms->gic, "revision", type); - qdev_prop_set_uint32(vms->gic, "num-cpu", smp_cpus); + qdev_prop_set_uint32(vms->gic, "num-cpu", num_cpus); /* Note that the num-irq property counts both internal and external * interrupts; there are always 32 of the former (mandated by GIC spec). */ @@ -846,7 +851,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) if (type == 3) { uint32_t redist0_capacity = vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE; - uint32_t redist0_count = MIN(smp_cpus, redist0_capacity); + uint32_t redist0_count = MIN(num_cpus, redist0_capacity); nb_redist_regions = virt_gicv3_redist_region_count(vms); @@ -867,7 +872,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) vms->memmap[VIRT_HIGH_GIC_REDIST2].size / GICV3_REDIST_SIZE; qdev_prop_set_uint32(vms->gic, "redist-region-count[1]", - MIN(smp_cpus - redist0_count, redist1_capacity)); + MIN(num_cpus - redist0_count, redist1_capacity)); } } else { if (!kvm_irqchip_in_kernel()) { @@ -894,7 +899,11 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) /* Wire the outputs from each CPU's generic timer and the GICv3 * maintenance interrupt signal to the appropriate GIC PPI inputs, - * and the GIC's IRQ/FIQ/VIRQ/VFIQ interrupt outputs to the CPU's inputs. + * and the GIC's IRQ/FIQ/VIRQ/VFIQ interrupt outputs to the CPU's + * inputs. + * + * The irqs of remaining CPUs (if we has) will be connected during + * hotplugging. */ for (i = 0; i < smp_cpus; i++) { connect_gic_cpu_irqs(vms, i); diff --git a/hw/intc/arm_gicv3.c b/hw/intc/arm_gicv3.c index 9591cfbcc0..864d4e4034 100644 --- a/hw/intc/arm_gicv3.c +++ b/hw/intc/arm_gicv3.c @@ -19,6 +19,7 @@ #include "qapi/error.h" #include "qemu/module.h" #include "hw/intc/arm_gicv3.h" +#include "hw/core/cpu.h" #include "gicv3_internal.h" static bool irqbetter(GICv3CPUState *cs, int irq, uint8_t prio) @@ -217,7 +218,9 @@ static void gicv3_update_noirqset(GICv3State *s, int start, int len) assert(len > 0); for (i = 0; i < s->num_cpu; i++) { - s->cpu[i].seenbetter = false; + if (qemu_get_cpu(i)) { + s->cpu[i].seenbetter = false; + } } /* Find the highest priority pending interrupt in this range. */ @@ -259,16 +262,18 @@ static void gicv3_update_noirqset(GICv3State *s, int start, int len) * now be the new best one). */ for (i = 0; i < s->num_cpu; i++) { - GICv3CPUState *cs = &s->cpu[i]; + if (qemu_get_cpu(i)) { + GICv3CPUState *cs = &s->cpu[i]; - if (cs->seenbetter) { - cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq); - } + if (cs->seenbetter) { + cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq); + } - if (!cs->seenbetter && cs->hppi.prio != 0xff && - cs->hppi.irq >= start && cs->hppi.irq < start + len) { - gicv3_full_update_noirqset(s); - break; + if (!cs->seenbetter && cs->hppi.prio != 0xff && + cs->hppi.irq >= start && cs->hppi.irq < start + len) { + gicv3_full_update_noirqset(s); + break; + } } } } @@ -279,7 +284,9 @@ void gicv3_update(GICv3State *s, int start, int len) gicv3_update_noirqset(s, start, len); for (i = 0; i < s->num_cpu; i++) { - gicv3_cpuif_update(&s->cpu[i]); + if (qemu_get_cpu(i)) { + gicv3_cpuif_update(&s->cpu[i]); + } } } @@ -291,7 +298,9 @@ void gicv3_full_update_noirqset(GICv3State *s) int i; for (i = 0; i < s->num_cpu; i++) { - s->cpu[i].hppi.prio = 0xff; + if (qemu_get_cpu(i)) { + s->cpu[i].hppi.prio = 0xff; + } } /* Note that we can guarantee that these functions will not @@ -302,7 +311,9 @@ void gicv3_full_update_noirqset(GICv3State *s) gicv3_update_noirqset(s, GIC_INTERNAL, s->num_irq - GIC_INTERNAL); for (i = 0; i < s->num_cpu; i++) { - gicv3_redist_update_noirqset(&s->cpu[i]); + if (qemu_get_cpu(i)) { + gicv3_redist_update_noirqset(&s->cpu[i]); + } } } @@ -315,7 +326,9 @@ void gicv3_full_update(GICv3State *s) gicv3_full_update_noirqset(s); for (i = 0; i < s->num_cpu; i++) { - gicv3_cpuif_update(&s->cpu[i]); + if (qemu_get_cpu(i)) { + gicv3_cpuif_update(&s->cpu[i]); + } } } @@ -406,7 +419,9 @@ static void arm_gic_realize(DeviceState *dev, Error **errp) gicv3_init_irqs_and_mmio(s, gicv3_set_irq, gic_ops); for (i = 0; i < s->num_cpu; i++) { - gicv3_cpu_realize(s, i); + if (qemu_get_cpu(i)) { + gicv3_cpu_realize(s, i); + } } } diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c index f8ef6817a4..a4976b2ba0 100644 --- a/hw/intc/arm_gicv3_common.c +++ b/hw/intc/arm_gicv3_common.c @@ -24,12 +24,14 @@ #include "qemu/osdep.h" #include "qapi/error.h" #include "qemu/module.h" +#include "qemu/error-report.h" #include "hw/core/cpu.h" #include "hw/intc/arm_gicv3_common.h" #include "hw/qdev-properties.h" #include "migration/vmstate.h" #include "gicv3_internal.h" #include "hw/arm/linux-boot-if.h" +#include "hw/boards.h" #include "sysemu/kvm.h" @@ -377,9 +379,14 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) for (i = 0; i < s->num_cpu; i++) { CPUState *cpu = qemu_get_cpu(i); + MachineState *ms = MACHINE(qdev_get_machine()); + MachineClass *mc = MACHINE_GET_CLASS(ms); + const CPUArchIdList *possible_cpus = NULL; uint64_t cpu_affid; - arm_gicv3_common_cpu_realize(s, i); + if (cpu) { + arm_gicv3_common_cpu_realize(s, i); + } /* Pre-construct the GICR_TYPER: * For our implementation: @@ -393,7 +400,18 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) * VLPIS == 0 (virtual LPIs not supported) * PLPIS == 0 (physical LPIs not supported) */ - cpu_affid = object_property_get_uint(OBJECT(cpu), "mp-affinity", NULL); + if (cpu) { + cpu_affid = object_property_get_uint(OBJECT(cpu), "mp-affinity", NULL); + } else { + if (!mc->possible_cpu_arch_ids) { + error_report("MachineClass must implement possible_cpu_arch_ids " + "hook to support pre-sizing GICv3"); + exit(1); + } + + possible_cpus = mc->possible_cpu_arch_ids(ms); + cpu_affid = possible_cpus->cpus[i].arch_id; + } /* The CPU mp-affinity property is in MPIDR register format; squash * the affinity bytes into 32 bits as the GICR_TYPER has them. diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c index 70809bcddd..274a40a40c 100644 --- a/hw/intc/arm_gicv3_cpuif.c +++ b/hw/intc/arm_gicv3_cpuif.c @@ -1676,6 +1676,10 @@ static void icc_generate_sgi(CPUARMState *env, GICv3CPUState *cs, aff, targetlist); for (i = 0; i < s->num_cpu; i++) { + if (!qemu_get_cpu(i)) { + continue; + } + GICv3CPUState *ocs = &s->cpu[i]; if (irm) { diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c index 95271e754b..2e2b08e31f 100644 --- a/hw/intc/arm_gicv3_kvm.c +++ b/hw/intc/arm_gicv3_kvm.c @@ -342,6 +342,10 @@ static void kvm_arm_gicv3_put(GICv3State *s) for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { GICv3CPUState *c = &s->cpu[ncpu]; + if (!qemu_get_cpu(ncpu)) { + continue; + } + reg64 = c->gicr_propbaser; regl = (uint32_t)reg64; kvm_gicr_access(s, GICR_PROPBASER, ncpu, ®l, true); @@ -361,6 +365,10 @@ static void kvm_arm_gicv3_put(GICv3State *s) for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { GICv3CPUState *c = &s->cpu[ncpu]; + if (!qemu_get_cpu(ncpu)) { + continue; + } + reg = c->gicr_ctlr; kvm_gicr_access(s, GICR_CTLR, ncpu, ®, true); @@ -457,6 +465,10 @@ static void kvm_arm_gicv3_put(GICv3State *s) GICv3CPUState *c = &s->cpu[ncpu]; int num_pri_bits; + if (!qemu_get_cpu(ncpu)) { + continue; + } + kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, true); kvm_gicc_access(s, ICC_CTLR_EL1, ncpu, &c->icc_ctlr_el1[GICV3_NS], true); @@ -524,6 +536,10 @@ static void kvm_arm_gicv3_get(GICv3State *s) /* Redistributor state (one per CPU) */ for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { + if (!qemu_get_cpu(ncpu)) { + continue; + } + GICv3CPUState *c = &s->cpu[ncpu]; kvm_gicr_access(s, GICR_CTLR, ncpu, ®, false); @@ -559,6 +575,10 @@ static void kvm_arm_gicv3_get(GICv3State *s) if (redist_typer & GICR_TYPER_PLPIS) { for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { + if (!qemu_get_cpu(ncpu)) { + continue; + } + GICv3CPUState *c = &s->cpu[ncpu]; kvm_gicr_access(s, GICR_PROPBASER, ncpu, ®l, false); @@ -612,6 +632,10 @@ static void kvm_arm_gicv3_get(GICv3State *s) */ for (ncpu = 0; ncpu < s->num_cpu; ncpu++) { + if (!qemu_get_cpu(ncpu)) { + continue; + } + GICv3CPUState *c = &s->cpu[ncpu]; int num_pri_bits; @@ -805,7 +829,9 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp) gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL); for (i = 0; i < s->num_cpu; i++) { - kvm_arm_gicv3_cpu_realize(s, i); + if (qemu_get_cpu(i)) { + kvm_arm_gicv3_cpu_realize(s, i); + } } /* Try to create the device via the device control API */ diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h index 2a838620d8..947d41f767 100644 --- a/include/hw/arm/virt.h +++ b/include/hw/arm/virt.h @@ -196,8 +196,9 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE; assert(vms->gic_version == VIRT_GIC_VERSION_3); + GICv3State *s = ARM_GICV3_COMMON(vms->gic); - return MACHINE(vms)->smp.cpus > redist0_capacity ? 2 : 1; + return s->num_cpu > redist0_capacity ? 2 : 1; } #endif /* QEMU_ARM_VIRT_H */ -- 2.27.0