memory: extract first iteration of address_space_read and address_space_write

We want to inline the case where there is only one iteration, because
then the compiler can also inline the memcpy. As a start, extract
everything after the first address_space_translate call.

Backports commit a203ac702e0720135fac8b1f2061d119814c1798 from qemu

qemu/aarch64.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_aarch64
 #define address_space_map address_space_map_aarch64
 #define address_space_read address_space_read_aarch64
+#define address_space_read_continue address_space_read_continue_aarch64
 #define address_space_rw address_space_rw_aarch64
 #define address_space_translate address_space_translate_aarch64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_aarch64

qemu/aarch64eb.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_aarch64eb
 #define address_space_map address_space_map_aarch64eb
 #define address_space_read address_space_read_aarch64eb
+#define address_space_read_continue address_space_read_continue_aarch64eb
 #define address_space_rw address_space_rw_aarch64eb
 #define address_space_translate address_space_translate_aarch64eb
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_aarch64eb

qemu/arm.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_arm
 #define address_space_map address_space_map_arm
 #define address_space_read address_space_read_arm
+#define address_space_read_continue address_space_read_continue_arm
 #define address_space_rw address_space_rw_arm
 #define address_space_translate address_space_translate_arm
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_arm

qemu/armeb.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_armeb
 #define address_space_map address_space_map_armeb
 #define address_space_read address_space_read_armeb
+#define address_space_read_continue address_space_read_continue_armeb
 #define address_space_rw address_space_rw_armeb
 #define address_space_translate address_space_translate_armeb
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_armeb

qemu/exec.c

@@ -1795,24 +1795,19 @@ static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
     return l;
 }
 
-MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
-                                const uint8_t *buf, int len)
+                                                MemTxAttrs attrs,
+                                                const uint8_t *buf,
+                                                int len, hwaddr addr1,
+                                                hwaddr l, MemoryRegion *mr)
 {
-    hwaddr l;
     uint8_t *ptr;
     uint64_t val;
-    hwaddr addr1;
-    MemoryRegion *mr;
     MemTxResult result = MEMTX_OK;
     // Unicorn: commented out
     //bool release_lock = false;
 
-    // Unicorn: commented out
+    for (;;) {
-    //rcu_read_lock();
-    while (len > 0) {
-        l = len;
-
-        mr = address_space_translate(as, addr, &addr1, &l, true);
         if (!mr)
             return true;
 
@@ -1867,6 +1862,13 @@ MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
         len -= l;
         buf += l;
         addr += l;
+
+        if (!len) {
+            break;
+        }
+
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, true);
     }
     // Unicorn: commented out
     //rcu_read_unlock();
@@ -1874,24 +1876,40 @@ MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
     return result;
 }
 
-MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
-                               uint8_t *buf, int len)
+                                const uint8_t *buf, int len)
 {
     hwaddr l;
-    uint8_t *ptr;
-    uint64_t val;
     hwaddr addr1;
     MemoryRegion *mr;
     MemTxResult result = MEMTX_OK;
+
+    if (len > 0) {
+        // Unicorn: commented out
+        //rcu_read_lock();
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, true);
+        result = address_space_write_continue(as, addr, attrs, buf, len,
+                                              addr1, l, mr);
+        // Unicorn: commented out
+        //rcu_read_unlock();
+    }
+
+    return result;
+}
+
+MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
+                                        MemTxAttrs attrs, uint8_t *buf,
+                                        int len, hwaddr addr1, hwaddr l,
+                                        MemoryRegion *mr)
+{
+    uint8_t *ptr;
+    uint64_t val;
+    MemTxResult result = MEMTX_OK;
     // Unicorn: commented out
     //bool release_lock = false;
 
-    // Unicorn: commented out
+    for (;;) {
-    //rcu_read_lock();
-    while (len > 0) {
-        l = len;
-        mr = address_space_translate(as, addr, &addr1, &l, false);
-
         if (!memory_access_is_direct(mr, false)) {
             /* I/O case */
             // Unicorn: commented out
@@ -1940,11 +1958,36 @@ MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
         len -= l;
         buf += l;
         addr += l;
+
+        if (!len) {
+            break;
+        }
+
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, false);
     }
 
-    // Unicorn: commented out
+    return result;
-    //rcu_read_unlock();
+}
 
+MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                               uint8_t *buf, int len)
+{
+    hwaddr l;
+    hwaddr addr1;
+    MemoryRegion *mr;
+    MemTxResult result = MEMTX_OK;
+
+    if (len > 0) {
+        // Unicorn: commented out
+        //rcu_read_lock();
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, false);
+        result = address_space_read_continue(as, addr, attrs, buf, len,
+                                             addr1, l, mr);
+        // Unicorn: commented out
+        //rcu_read_unlock();
+    }
     return result;
 }
 

qemu/header_gen.py

@@ -99,6 +99,7 @@ symbols = (
     'address_space_lookup_region',
     'address_space_map',
     'address_space_read',
+    'address_space_read_continue',
     'address_space_rw',
     'address_space_translate',
     'address_space_translate_for_iotlb',

qemu/include/exec/memory.h

@@ -1077,7 +1077,6 @@ void *address_space_map(AddressSpace *as, hwaddr addr,
 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
                          int is_write, hwaddr access_len);
 
-
 void memory_register_types(struct uc_struct *uc);
 
 MemoryRegion *memory_map(struct uc_struct *uc, hwaddr begin, size_t size, uint32_t perms);
@@ -1085,6 +1084,12 @@ MemoryRegion *memory_map_ptr(struct uc_struct *uc, hwaddr begin, size_t size, ui
 void memory_unmap(struct uc_struct *uc, MemoryRegion *mr);
 int memory_free(struct uc_struct *uc);
 
+/* Internal functions, part of the implementation of address_space_read.  */
+MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
+                                        MemTxAttrs attrs, uint8_t *buf,
+                                        int len, hwaddr addr1, hwaddr l,
+                    MemoryRegion *mr);
+
 #endif
 
 #endif

qemu/m68k.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_m68k
 #define address_space_map address_space_map_m68k
 #define address_space_read address_space_read_m68k
+#define address_space_read_continue address_space_read_continue_m68k
 #define address_space_rw address_space_rw_m68k
 #define address_space_translate address_space_translate_m68k
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_m68k

qemu/mips.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_mips
 #define address_space_map address_space_map_mips
 #define address_space_read address_space_read_mips
+#define address_space_read_continue address_space_read_continue_mips
 #define address_space_rw address_space_rw_mips
 #define address_space_translate address_space_translate_mips
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips

qemu/mips64.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_mips64
 #define address_space_map address_space_map_mips64
 #define address_space_read address_space_read_mips64
+#define address_space_read_continue address_space_read_continue_mips64
 #define address_space_rw address_space_rw_mips64
 #define address_space_translate address_space_translate_mips64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips64

qemu/mips64el.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_mips64el
 #define address_space_map address_space_map_mips64el
 #define address_space_read address_space_read_mips64el
+#define address_space_read_continue address_space_read_continue_mips64el
 #define address_space_rw address_space_rw_mips64el
 #define address_space_translate address_space_translate_mips64el
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips64el

qemu/mipsel.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_mipsel
 #define address_space_map address_space_map_mipsel
 #define address_space_read address_space_read_mipsel
+#define address_space_read_continue address_space_read_continue_mipsel
 #define address_space_rw address_space_rw_mipsel
 #define address_space_translate address_space_translate_mipsel
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mipsel

qemu/powerpc.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_powerpc
 #define address_space_map address_space_map_powerpc
 #define address_space_read address_space_read_powerpc
+#define address_space_read_continue address_space_read_continue_powerpc
 #define address_space_rw address_space_rw_powerpc
 #define address_space_translate address_space_translate_powerpc
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_powerpc

qemu/sparc.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_sparc
 #define address_space_map address_space_map_sparc
 #define address_space_read address_space_read_sparc
+#define address_space_read_continue address_space_read_continue_sparc
 #define address_space_rw address_space_rw_sparc
 #define address_space_translate address_space_translate_sparc
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_sparc

qemu/sparc64.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_sparc64
 #define address_space_map address_space_map_sparc64
 #define address_space_read address_space_read_sparc64
+#define address_space_read_continue address_space_read_continue_sparc64
 #define address_space_rw address_space_rw_sparc64
 #define address_space_translate address_space_translate_sparc64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_sparc64

qemu/x86_64.h

@@ -93,6 +93,7 @@
 #define address_space_lookup_region address_space_lookup_region_x86_64
 #define address_space_map address_space_map_x86_64
 #define address_space_read address_space_read_x86_64
+#define address_space_read_continue address_space_read_continue_x86_64
 #define address_space_rw address_space_rw_x86_64
 #define address_space_translate address_space_translate_x86_64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_x86_64