Added an invalid instruction hook (#1132)

* first draft for an invalid instruction hook

* Fixed documentation on return value of invalid insn hook

Backports commit 07f94ad1fc62293cac330df9714d739be6354926 from unicorn

bindings/python/unicorn/unicorn.py

@@ -146,6 +146,7 @@ _uc.uc_hook_add = _uc.uc_hook_add
 _uc.uc_hook_add.restype = ucerr
 
 UC_HOOK_CODE_CB = ctypes.CFUNCTYPE(None, uc_engine, ctypes.c_uint64, ctypes.c_size_t, ctypes.c_void_p)
+UC_HOOK_INSN_INVALID_CB = ctypes.CFUNCTYPE(ctypes.c_bool, uc_engine, ctypes.c_void_p)
 UC_HOOK_MEM_INVALID_CB = ctypes.CFUNCTYPE(
     ctypes.c_bool, uc_engine, ctypes.c_int,
     ctypes.c_uint64, ctypes.c_int, ctypes.c_int64, ctypes.c_void_p
@@ -492,6 +493,11 @@ class Uc(object):
         (cb, data) = self._callbacks[user_data]
         cb(self, intno, data)
 
+    def _hook_insn_invalid_cb(self, handle, user_data):
+        # call user's callback with self object
+        (cb, data) = self._callbacks[user_data]
+        return cb(self, data)
+
     def _hook_insn_in_cb(self, handle, port, size, user_data):
         # call user's callback with self object
         (cb, data) = self._callbacks[user_data]
@@ -536,6 +542,13 @@ class Uc(object):
                 ctypes.cast(self._callback_count, ctypes.c_void_p),
                 ctypes.c_uint64(begin), ctypes.c_uint64(end)
             )
+        elif htype == uc.UC_HOOK_INSN_INVALID:
+            cb = ctypes.cast(UC_HOOK_INSN_INVALID_CB(self._hook_insn_invalid_cb), UC_HOOK_INSN_INVALID_CB)
+            status = _uc.uc_hook_add(
+                self._uch, ctypes.byref(_h2), htype, cb,
+                ctypes.cast(self._callback_count, ctypes.c_void_p),
+                ctypes.c_uint64(begin), ctypes.c_uint64(end)
+            )
         else:
             if htype in (uc.UC_HOOK_BLOCK, uc.UC_HOOK_CODE):
                 # set callback with wrapper, so it can be called

bindings/python/unicorn/unicorn_const.py

@@ -85,6 +85,7 @@ UC_HOOK_MEM_READ = 1024
 UC_HOOK_MEM_WRITE = 2048
 UC_HOOK_MEM_FETCH = 4096
 UC_HOOK_MEM_READ_AFTER = 8192
+UC_HOOK_INSN_INVALID = 16384
 UC_HOOK_MEM_UNMAPPED = 112
 UC_HOOK_MEM_PROT = 896
 UC_HOOK_MEM_READ_INVALID = 144

include/uc_priv.h

@@ -110,6 +110,7 @@ enum uc_hook_idx {
     UC_HOOK_MEM_WRITE_IDX,
     UC_HOOK_MEM_FETCH_IDX,
     UC_HOOK_MEM_READ_AFTER_IDX,
+    UC_HOOK_INSN_INVALID_IDX,
 
     UC_HOOK_MAX,
 };

include/unicorn/unicorn.h

@@ -179,6 +179,15 @@ typedef void (*uc_cb_hookcode_t)(uc_engine *uc, uint64_t address, uint32_t size,
 */
 typedef void (*uc_cb_hookintr_t)(uc_engine *uc, uint32_t intno, void *user_data);
 
+/*
+  Callback function for tracing invalid instructions
+
+  @user_data: user data passed to tracing APIs.
+
+  @return: return true to continue, or false to stop program (due to invalid instruction).
+*/
+typedef bool (*uc_cb_hookinsn_invalid_t)(uc_engine *uc, void *user_data);
+
 /*
   Callback function for tracing IN instruction of X86
 
@@ -242,6 +251,8 @@ typedef enum uc_hook_type {
     // Hook memory read events, but only successful access.
     // The callback will be triggered after successful read.
     UC_HOOK_MEM_READ_AFTER = 1 << 13,
+    // Hook invalid instructions exceptions.
+    UC_HOOK_INSN_INVALID = 1 << 14,
 } uc_hook_type;
 
 // Hook type for all events of unmapped memory access

qemu/accel/tcg/cpu-exec.c

@@ -294,13 +294,6 @@ static inline bool cpu_handle_exception(struct uc_struct *uc, CPUState *cpu, int
     struct hook *hook;
 
     if (cpu->exception_index >= 0) {
-        if (uc->stop_interrupt && uc->stop_interrupt(cpu->exception_index)) {
-            cpu->halted = 1;
-            uc->invalid_error = UC_ERR_INSN_INVALID;
-            *ret = EXCP_HLT;
-            return true;
-        }
-
         if (cpu->exception_index >= EXCP_INTERRUPT) {
             /* exit request from the cpu execution loop */
             *ret = cpu->exception_index;
@@ -323,16 +316,33 @@ static inline bool cpu_handle_exception(struct uc_struct *uc, CPUState *cpu, int
             return true;
 #else
             bool catched = false;
-            // Unicorn: call registered interrupt callbacks
+            if (uc->stop_interrupt && uc->stop_interrupt(cpu->exception_index)) {
-            HOOK_FOREACH_VAR_DECLARE;
+                // Unicorn: call registered invalid instruction callbacks
-            HOOK_FOREACH(uc, hook, UC_HOOK_INTR) {
+                HOOK_FOREACH_VAR_DECLARE;
-                ((uc_cb_hookintr_t)hook->callback)(uc, cpu->exception_index, hook->user_data);
+                HOOK_FOREACH(uc, hook, UC_HOOK_INSN_INVALID) {
-                catched = true;
+                    catched = ((uc_cb_hookinsn_invalid_t)hook->callback)(uc, hook->user_data);
+                    if (catched) {
+                        break;
+                    }
+                }
+                if (!catched) {
+                    uc->invalid_error = UC_ERR_INSN_INVALID;
+                }
+            } else {
+                // Unicorn: call registered interrupt callbacks
+                HOOK_FOREACH_VAR_DECLARE;
+                HOOK_FOREACH(uc, hook, UC_HOOK_INTR) {
+                    ((uc_cb_hookintr_t)hook->callback)(uc, cpu->exception_index, hook->user_data);
+                    catched = true;
+                }
+                if (!catched) {
+                    uc->invalid_error = UC_ERR_EXCEPTION;
+                }
             }
+
             // Unicorn: If un-catched interrupt, stop executions.
             if (!catched) {
                 cpu->halted = 1;
-                uc->invalid_error = UC_ERR_EXCEPTION;
                 *ret = EXCP_HLT;
                 return true;
             }