From 309b85548fbf5c31bb6dda8fcd4484763308ca47 Mon Sep 17 00:00:00 2001
From: Bharata B Rao <bharata@linux.vnet.ibm.com>
Date: Wed, 21 Mar 2018 07:57:39 -0400
Subject: [PATCH] cpu: Convert cpu_index into a bitmap

Currently CPUState::cpu_index is monotonically increasing and a newly
created CPU always gets the next higher index. The next available
index is calculated by counting the existing number of CPUs. This is
fine as long as we only add CPUs, but there are architectures which
are starting to support CPU removal, too. For an architecture like PowerPC
which derives its CPU identifier (device tree ID) from cpu_index, the
existing logic of generating cpu_index values causes problems.

With the currently proposed method of handling vCPU removal by parking
the vCPU fd in QEMU
(Ref: http://lists.gnu.org/archive/html/qemu-devel/2015-02/msg02604.html),
generating cpu_index this way will not work for PowerPC.

This patch changes the way cpu_index is handed out by maintaining
a bit map of the CPUs that tracks both addition and removal of CPUs.

The CPU bitmap allocation logic is part of cpu_exec_init(), which is
called by instance_init routines of various CPU targets. Newly added
cpu_exec_exit() API handles the deallocation part and this routine is
called from generic CPU instance_finalize.

Note: This new CPU enumeration is for !CONFIG_USER_ONLY only.
CONFIG_USER_ONLY continues to have the old enumeration logic.

Backports commit b7bca7333411bd19c449147e8202ae6b0e4a8e09 from qemu
---
 include/uc_priv.h      |  2 ++
 qemu/aarch64.h         |  1 +
 qemu/aarch64eb.h       |  1 +
 qemu/arm.h             |  1 +
 qemu/armeb.h           |  1 +
 qemu/exec.c            | 62 +++++++++++++++++++++++++++++++++++++++++-
 qemu/header_gen.py     |  1 +
 qemu/include/qom/cpu.h |  1 +
 qemu/m68k.h            |  1 +
 qemu/mips.h            |  1 +
 qemu/mips64.h          |  1 +
 qemu/mips64el.h        |  1 +
 qemu/mipsel.h          |  1 +
 qemu/powerpc.h         |  1 +
 qemu/qom/cpu.c         |  8 +++++-
 qemu/sparc.h           |  1 +
 qemu/sparc64.h         |  1 +
 qemu/unicorn_common.h  |  1 +
 qemu/x86_64.h          |  1 +
 19 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/include/uc_priv.h b/include/uc_priv.h
index d235e3e9..2a726cee 100644
--- a/include/uc_priv.h
+++ b/include/uc_priv.h
@@ -58,6 +58,7 @@ typedef void (*uc_args_void_t)(void*);
 
 typedef void (*uc_args_uc_t)(struct uc_struct*);
 typedef int (*uc_args_int_uc_t)(struct uc_struct*);
+typedef void (*uc_cpu_exec_exit_t)(CPUState*);
 
 typedef bool (*uc_args_tcg_enable_t)(struct uc_struct*);
 
@@ -162,6 +163,7 @@ struct uc_struct {
     uc_args_int_t stop_interrupt;   // check if the interrupt should stop emulation
 
     uc_args_uc_t init_arch, cpu_exec_init_all;
+    uc_cpu_exec_exit_t cpu_exec_exit;
     uc_args_int_uc_t vm_start;
     uc_args_tcg_enable_t tcg_enabled;
     uc_args_uc_long_t tcg_exec_init;
diff --git a/qemu/aarch64.h b/qemu/aarch64.h
index d0ea77a8..6b8004dc 100644
--- a/qemu/aarch64.h
+++ b/qemu/aarch64.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_aarch64
 #define cpu_dump_statistics cpu_dump_statistics_aarch64
 #define cpu_exec cpu_exec_aarch64
+#define cpu_exec_exit cpu_exec_exit_aarch64
 #define cpu_exec_init cpu_exec_init_aarch64
 #define cpu_exec_init_all cpu_exec_init_all_aarch64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_aarch64
diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h
index d6a79ac7..9205a359 100644
--- a/qemu/aarch64eb.h
+++ b/qemu/aarch64eb.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_aarch64eb
 #define cpu_dump_statistics cpu_dump_statistics_aarch64eb
 #define cpu_exec cpu_exec_aarch64eb
+#define cpu_exec_exit cpu_exec_exit_aarch64eb
 #define cpu_exec_init cpu_exec_init_aarch64eb
 #define cpu_exec_init_all cpu_exec_init_all_aarch64eb
 #define cpu_exec_step_atomic cpu_exec_step_atomic_aarch64eb
diff --git a/qemu/arm.h b/qemu/arm.h
index bd0a792c..2c6d61ca 100644
--- a/qemu/arm.h
+++ b/qemu/arm.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_arm
 #define cpu_dump_statistics cpu_dump_statistics_arm
 #define cpu_exec cpu_exec_arm
+#define cpu_exec_exit cpu_exec_exit_arm
 #define cpu_exec_init cpu_exec_init_arm
 #define cpu_exec_init_all cpu_exec_init_all_arm
 #define cpu_exec_step_atomic cpu_exec_step_atomic_arm
diff --git a/qemu/armeb.h b/qemu/armeb.h
index 2dcb704b..6ba9e124 100644
--- a/qemu/armeb.h
+++ b/qemu/armeb.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_armeb
 #define cpu_dump_statistics cpu_dump_statistics_armeb
 #define cpu_exec cpu_exec_armeb
+#define cpu_exec_exit cpu_exec_exit_armeb
 #define cpu_exec_init cpu_exec_init_armeb
 #define cpu_exec_init_all cpu_exec_init_all_armeb
 #define cpu_exec_step_atomic cpu_exec_step_atomic_armeb
diff --git a/qemu/exec.c b/qemu/exec.c
index aa4c9ebe..6d95c776 100644
--- a/qemu/exec.c
+++ b/qemu/exec.c
@@ -577,18 +577,78 @@ AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
 }
 #endif
 
+#ifndef CONFIG_USER_ONLY
+static int cpu_get_free_index(Error **errp)
+{
+    // Unicorn: if'd out
+#if 0
+    int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
+
+    if (cpu >= MAX_CPUMASK_BITS) {
+        error_setg(errp, "Trying to use more CPUs than max of %d",
+                   MAX_CPUMASK_BITS);
+        return -1;
+    }
+
+    bitmap_set(cpu_index_map, cpu, 1);
+    return cpu;
+#endif
+    return 0;
+}
+
+void cpu_exec_exit(CPUState *cpu)
+{
+    if (cpu->cpu_index == -1) {
+        /* cpu_index was never allocated by this @cpu or was already freed. */
+        return;
+    }
+
+    // Unicorn: if'd out
+#if 0
+    bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
+#endif
+    cpu->cpu_index = -1;
+}
+#else
+
+static int cpu_get_free_index(Error **errp)
+{
+    // Unicorn: if'd out
+#if 0
+    CPUState *some_cpu;
+    int cpu_index = 0;
+
+    CPU_FOREACH(some_cpu) {
+        cpu_index++;
+    }
+    return cpu_index;
+#endif
+    return 0;
+}
+
+void cpu_exec_exit(CPUState *cpu)
+{
+}
+#endif
+
 void cpu_exec_init(CPUState *cpu, Error **errp, void *opaque)
 {
     struct uc_struct *uc = opaque;
     CPUClass *cc = CPU_GET_CLASS(uc, cpu);
     CPUArchState *env = cpu->env_ptr;
+    Error *local_err = NULL;
 
     cpu->as = NULL;
-    cpu->cpu_index = 0;
     cpu->num_ases = 0;
     cpu->uc = uc;
     env->uc = uc;
 
+    cpu->cpu_index = cpu_get_free_index(&local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     // TODO: assert uc does not already have a cpu?
     uc->cpu = cpu;
 
diff --git a/qemu/header_gen.py b/qemu/header_gen.py
index 8b8cbe89..fdafd258 100644
--- a/qemu/header_gen.py
+++ b/qemu/header_gen.py
@@ -282,6 +282,7 @@ symbols = (
     'cpu_common_reset',
     'cpu_dump_statistics',
     'cpu_exec',
+    'cpu_exec_exit',
     'cpu_exec_init',
     'cpu_exec_init_all',
     'cpu_exec_step_atomic',
diff --git a/qemu/include/qom/cpu.h b/qemu/include/qom/cpu.h
index 0345167c..50e6f0b3 100644
--- a/qemu/include/qom/cpu.h
+++ b/qemu/include/qom/cpu.h
@@ -807,6 +807,7 @@ static inline bool cpu_breakpoint_test(CPUState *cpu, vaddr pc, int mask)
 
 void QEMU_NORETURN cpu_abort(CPUState *cpu, const char *fmt, ...)
     GCC_FMT_ATTR(2, 3);
+void cpu_exec_exit(CPUState *cpu);
 
 void cpu_register_types(struct uc_struct *uc);
 
diff --git a/qemu/m68k.h b/qemu/m68k.h
index ecd7499b..32513a5f 100644
--- a/qemu/m68k.h
+++ b/qemu/m68k.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_m68k
 #define cpu_dump_statistics cpu_dump_statistics_m68k
 #define cpu_exec cpu_exec_m68k
+#define cpu_exec_exit cpu_exec_exit_m68k
 #define cpu_exec_init cpu_exec_init_m68k
 #define cpu_exec_init_all cpu_exec_init_all_m68k
 #define cpu_exec_step_atomic cpu_exec_step_atomic_m68k
diff --git a/qemu/mips.h b/qemu/mips.h
index d66d73a6..8b539661 100644
--- a/qemu/mips.h
+++ b/qemu/mips.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mips
 #define cpu_dump_statistics cpu_dump_statistics_mips
 #define cpu_exec cpu_exec_mips
+#define cpu_exec_exit cpu_exec_exit_mips
 #define cpu_exec_init cpu_exec_init_mips
 #define cpu_exec_init_all cpu_exec_init_all_mips
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mips
diff --git a/qemu/mips64.h b/qemu/mips64.h
index 42e318b3..9010919b 100644
--- a/qemu/mips64.h
+++ b/qemu/mips64.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mips64
 #define cpu_dump_statistics cpu_dump_statistics_mips64
 #define cpu_exec cpu_exec_mips64
+#define cpu_exec_exit cpu_exec_exit_mips64
 #define cpu_exec_init cpu_exec_init_mips64
 #define cpu_exec_init_all cpu_exec_init_all_mips64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mips64
diff --git a/qemu/mips64el.h b/qemu/mips64el.h
index b281b9aa..a2e44acd 100644
--- a/qemu/mips64el.h
+++ b/qemu/mips64el.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mips64el
 #define cpu_dump_statistics cpu_dump_statistics_mips64el
 #define cpu_exec cpu_exec_mips64el
+#define cpu_exec_exit cpu_exec_exit_mips64el
 #define cpu_exec_init cpu_exec_init_mips64el
 #define cpu_exec_init_all cpu_exec_init_all_mips64el
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mips64el
diff --git a/qemu/mipsel.h b/qemu/mipsel.h
index 81df74e8..6ff8943e 100644
--- a/qemu/mipsel.h
+++ b/qemu/mipsel.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mipsel
 #define cpu_dump_statistics cpu_dump_statistics_mipsel
 #define cpu_exec cpu_exec_mipsel
+#define cpu_exec_exit cpu_exec_exit_mipsel
 #define cpu_exec_init cpu_exec_init_mipsel
 #define cpu_exec_init_all cpu_exec_init_all_mipsel
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mipsel
diff --git a/qemu/powerpc.h b/qemu/powerpc.h
index 4a0946bf..76036bd9 100644
--- a/qemu/powerpc.h
+++ b/qemu/powerpc.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_powerpc
 #define cpu_dump_statistics cpu_dump_statistics_powerpc
 #define cpu_exec cpu_exec_powerpc
+#define cpu_exec_exit cpu_exec_exit_powerpc
 #define cpu_exec_init cpu_exec_init_powerpc
 #define cpu_exec_init_all cpu_exec_init_all_powerpc
 #define cpu_exec_step_atomic cpu_exec_step_atomic_powerpc
diff --git a/qemu/qom/cpu.c b/qemu/qom/cpu.c
index 9a7cbb6b..b79fd8a0 100644
--- a/qemu/qom/cpu.c
+++ b/qemu/qom/cpu.c
@@ -258,10 +258,16 @@ static void cpu_common_initfn(struct uc_struct *uc, Object *obj, void *opaque)
 {
     CPUState *cpu = CPU(obj);
 
+    cpu->cpu_index = -1;
     QTAILQ_INIT(&cpu->breakpoints);
     QTAILQ_INIT(&cpu->watchpoints);
 }
 
+static void cpu_common_finalize(struct uc_struct *uc, Object *obj, void *opaque)
+{
+    uc->cpu_exec_exit(CPU(obj));
+}
+
 static int64_t cpu_common_get_arch_id(CPUState *cpu)
 {
     return cpu->cpu_index;
@@ -308,7 +314,7 @@ static const TypeInfo cpu_type_info = {
 
     cpu_common_initfn,
     NULL,
-    NULL,
+    cpu_common_finalize,
 
     NULL,
 
diff --git a/qemu/sparc.h b/qemu/sparc.h
index f705f196..2b96df45 100644
--- a/qemu/sparc.h
+++ b/qemu/sparc.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_sparc
 #define cpu_dump_statistics cpu_dump_statistics_sparc
 #define cpu_exec cpu_exec_sparc
+#define cpu_exec_exit cpu_exec_exit_sparc
 #define cpu_exec_init cpu_exec_init_sparc
 #define cpu_exec_init_all cpu_exec_init_all_sparc
 #define cpu_exec_step_atomic cpu_exec_step_atomic_sparc
diff --git a/qemu/sparc64.h b/qemu/sparc64.h
index bca41e2c..9961a6ce 100644
--- a/qemu/sparc64.h
+++ b/qemu/sparc64.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_sparc64
 #define cpu_dump_statistics cpu_dump_statistics_sparc64
 #define cpu_exec cpu_exec_sparc64
+#define cpu_exec_exit cpu_exec_exit_sparc64
 #define cpu_exec_init cpu_exec_init_sparc64
 #define cpu_exec_init_all cpu_exec_init_all_sparc64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_sparc64
diff --git a/qemu/unicorn_common.h b/qemu/unicorn_common.h
index cba7f339..71adf268 100644
--- a/qemu/unicorn_common.h
+++ b/qemu/unicorn_common.h
@@ -100,6 +100,7 @@ static inline void uc_common_init(struct uc_struct* uc)
     uc->tcg_enabled = tcg_enabled;
     uc->tcg_exec_init = tcg_exec_init;
     uc->cpu_exec_init_all = cpu_exec_init_all;
+    uc->cpu_exec_exit = cpu_exec_exit;
     uc->vm_start = vm_start;
     uc->memory_map = memory_map;
     uc->memory_map_ptr = memory_map_ptr;
diff --git a/qemu/x86_64.h b/qemu/x86_64.h
index 26eb361e..d987b500 100644
--- a/qemu/x86_64.h
+++ b/qemu/x86_64.h
@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_x86_64
 #define cpu_dump_statistics cpu_dump_statistics_x86_64
 #define cpu_exec cpu_exec_x86_64
+#define cpu_exec_exit cpu_exec_exit_x86_64
 #define cpu_exec_init cpu_exec_init_x86_64
 #define cpu_exec_init_all cpu_exec_init_all_x86_64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_x86_64