From 1650af8c8bf9b67a6ab7f5e2d1b4fcd44680cd89 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 17 Feb 2018 20:43:00 -0500 Subject: [PATCH] memory: try to inline constant-length reads memcpy can take a large amount of time for small reads and writes. Handle the common case of reading s/g descriptors from memory (there is no corresponding "write" case that is as common, because writes often use address_space_st* functions) by inlining the relevant parts of address_space_read into the caller. Backports commit 3cc8f884996584630734a90c9b3c535af81e3c92 from qemu --- qemu/aarch64.h | 2 +- qemu/aarch64eb.h | 2 +- qemu/arm.h | 2 +- qemu/armeb.h | 2 +- qemu/exec.c | 15 +------- qemu/header_gen.py | 2 +- qemu/include/exec/memory.h | 75 ++++++++++++++++++++++++++++-------- qemu/include/exec/ram_addr.h | 1 - qemu/m68k.h | 2 +- qemu/mips.h | 2 +- qemu/mips64.h | 2 +- qemu/mips64el.h | 2 +- qemu/mipsel.h | 2 +- qemu/powerpc.h | 2 +- qemu/sparc.h | 2 +- qemu/sparc64.h | 2 +- qemu/x86_64.h | 2 +- 17 files changed, 76 insertions(+), 43 deletions(-) diff --git a/qemu/aarch64.h b/qemu/aarch64.h index 6059a4fe..58b12105 100644 --- a/qemu/aarch64.h +++ b/qemu/aarch64.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_aarch64 #define address_space_read address_space_read_aarch64 #define address_space_read_continue address_space_read_continue_aarch64 +#define address_space_read_full address_space_read_full_aarch64 #define address_space_rw address_space_rw_aarch64 #define address_space_translate address_space_translate_aarch64 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_aarch64 @@ -2040,7 +2041,6 @@ #define mem_add mem_add_aarch64 #define mem_begin mem_begin_aarch64 #define mem_commit mem_commit_aarch64 -#define memory_access_is_direct memory_access_is_direct_aarch64 #define memory_access_size memory_access_size_aarch64 #define memory_init memory_init_aarch64 #define memory_listener_match memory_listener_match_aarch64 diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h index 53c4a851..1f2a8675 100644 --- a/qemu/aarch64eb.h +++ b/qemu/aarch64eb.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_aarch64eb #define address_space_read address_space_read_aarch64eb #define address_space_read_continue address_space_read_continue_aarch64eb +#define address_space_read_full address_space_read_full_aarch64eb #define address_space_rw address_space_rw_aarch64eb #define address_space_translate address_space_translate_aarch64eb #define address_space_translate_for_iotlb address_space_translate_for_iotlb_aarch64eb @@ -2040,7 +2041,6 @@ #define mem_add mem_add_aarch64eb #define mem_begin mem_begin_aarch64eb #define mem_commit mem_commit_aarch64eb -#define memory_access_is_direct memory_access_is_direct_aarch64eb #define memory_access_size memory_access_size_aarch64eb #define memory_init memory_init_aarch64eb #define memory_listener_match memory_listener_match_aarch64eb diff --git a/qemu/arm.h b/qemu/arm.h index 590e2240..b67bee1f 100644 --- a/qemu/arm.h +++ b/qemu/arm.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_arm #define address_space_read address_space_read_arm #define address_space_read_continue address_space_read_continue_arm +#define address_space_read_full address_space_read_full_arm #define address_space_rw address_space_rw_arm #define address_space_translate address_space_translate_arm #define address_space_translate_for_iotlb address_space_translate_for_iotlb_arm @@ -2040,7 +2041,6 @@ #define mem_add mem_add_arm #define mem_begin mem_begin_arm #define mem_commit mem_commit_arm -#define memory_access_is_direct memory_access_is_direct_arm #define memory_access_size memory_access_size_arm #define memory_init memory_init_arm #define memory_listener_match memory_listener_match_arm diff --git a/qemu/armeb.h b/qemu/armeb.h index 70aebeba..c780561a 100644 --- a/qemu/armeb.h +++ b/qemu/armeb.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_armeb #define address_space_read address_space_read_armeb #define address_space_read_continue address_space_read_continue_armeb +#define address_space_read_full address_space_read_full_armeb #define address_space_rw address_space_rw_armeb #define address_space_translate address_space_translate_armeb #define address_space_translate_for_iotlb address_space_translate_for_iotlb_armeb @@ -2040,7 +2041,6 @@ #define mem_add mem_add_armeb #define mem_begin mem_begin_armeb #define mem_commit mem_commit_armeb -#define memory_access_is_direct memory_access_is_direct_armeb #define memory_access_size memory_access_size_armeb #define memory_init memory_init_armeb #define memory_listener_match memory_listener_match_armeb diff --git a/qemu/exec.c b/qemu/exec.c index 252dfa6b..d5445694 100644 --- a/qemu/exec.c +++ b/qemu/exec.c @@ -326,17 +326,6 @@ address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *x return section; } -static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write) -{ - if (is_write) { - return memory_region_is_ram(mr) && !mr->readonly; - } else { - return memory_region_is_ram(mr) || memory_region_is_romd(mr); - } - - return false; -} - MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr, hwaddr *xlat, hwaddr *plen, bool is_write) @@ -1970,8 +1959,8 @@ MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr, return result; } -MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, - uint8_t *buf, int len) +MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len) { hwaddr l; hwaddr addr1; diff --git a/qemu/header_gen.py b/qemu/header_gen.py index b0964d48..44b5ccc2 100644 --- a/qemu/header_gen.py +++ b/qemu/header_gen.py @@ -100,6 +100,7 @@ symbols = ( 'address_space_map', 'address_space_read', 'address_space_read_continue', + 'address_space_read_full', 'address_space_rw', 'address_space_translate', 'address_space_translate_for_iotlb', @@ -2046,7 +2047,6 @@ symbols = ( 'mem_add', 'mem_begin', 'mem_commit', - 'memory_access_is_direct', 'memory_access_size', 'memory_init', 'memory_listener_match', diff --git a/qemu/include/exec/memory.h b/qemu/include/exec/memory.h index 63a01649..893249ab 100644 --- a/qemu/include/exec/memory.h +++ b/qemu/include/exec/memory.h @@ -946,21 +946,6 @@ MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, const uint8_t *buf, int len); -/** - * address_space_read: read from an address space. - * - * Return a MemTxResult indicating whether the operation succeeded - * or failed (eg unassigned memory, device rejected the transaction, - * IOMMU fault). - * - * @as: #AddressSpace to be accessed - * @addr: address within that address space - * @attrs: memory transaction attributes - * @buf: buffer with the data transferred - */ -MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, - uint8_t *buf, int len); - /** * address_space_ld*: load from an address space * address_space_st*: store to an address space @@ -1102,6 +1087,66 @@ MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr, int len, hwaddr addr1, hwaddr l, MemoryRegion *mr); +MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len); +void *qemu_get_ram_ptr(struct uc_struct *uc, ram_addr_t addr); + +static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write) +{ + if (is_write) { + return memory_region_is_ram(mr) && !mr->readonly; + } else { + return memory_region_is_ram(mr) || memory_region_is_romd(mr); + } + + return false; +} + +/** + * address_space_read: read from an address space. + * + * Return a MemTxResult indicating whether the operation succeeded + * or failed (eg unassigned memory, device rejected the transaction, + * IOMMU fault). + * + * @as: #AddressSpace to be accessed + * @addr: address within that address space + * @attrs: memory transaction attributes + * @buf: buffer with the data transferred + */ +static inline +MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, + uint8_t *buf, int len) +{ + MemTxResult result = MEMTX_OK; + /* Unicorn: commented out + hwaddr l, addr1; + void *ptr; + MemoryRegion *mr; + + if (__builtin_constant_p(len)) { + if (len) { + // Unicorn: commented out + //rcu_read_lock(); + l = len; + mr = address_space_translate(as, addr, &addr1, &l, false); + if (len == l && memory_access_is_direct(mr, false)) { + addr1 += memory_region_get_ram_addr(mr); + ptr = qemu_get_ram_ptr(addr1); + memcpy(buf, ptr, len); + } else { + result = address_space_read_continue(as, addr, attrs, buf, len, + addr1, l, mr); + } + // Unicorn: commented out + //rcu_read_unlock(); + } + } else {*/ + result = address_space_read_full(as, addr, attrs, buf, len); + //} + return result; +} + #endif #endif diff --git a/qemu/include/exec/ram_addr.h b/qemu/include/exec/ram_addr.h index 164f566d..ca9e41b7 100644 --- a/qemu/include/exec/ram_addr.h +++ b/qemu/include/exec/ram_addr.h @@ -33,7 +33,6 @@ ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, MemoryRegion *mr, Error **errp); int qemu_get_ram_fd(struct uc_struct *uc, ram_addr_t addr); void *qemu_get_ram_block_host_ptr(struct uc_struct *uc, ram_addr_t addr); -void *qemu_get_ram_ptr(struct uc_struct *uc, ram_addr_t addr); void qemu_ram_free(struct uc_struct *c, ram_addr_t addr); int qemu_ram_resize(struct uc_struct *c, ram_addr_t base, ram_addr_t newsize, Error **errp); diff --git a/qemu/m68k.h b/qemu/m68k.h index fcbdaeb2..ebe93e9a 100644 --- a/qemu/m68k.h +++ b/qemu/m68k.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_m68k #define address_space_read address_space_read_m68k #define address_space_read_continue address_space_read_continue_m68k +#define address_space_read_full address_space_read_full_m68k #define address_space_rw address_space_rw_m68k #define address_space_translate address_space_translate_m68k #define address_space_translate_for_iotlb address_space_translate_for_iotlb_m68k @@ -2040,7 +2041,6 @@ #define mem_add mem_add_m68k #define mem_begin mem_begin_m68k #define mem_commit mem_commit_m68k -#define memory_access_is_direct memory_access_is_direct_m68k #define memory_access_size memory_access_size_m68k #define memory_init memory_init_m68k #define memory_listener_match memory_listener_match_m68k diff --git a/qemu/mips.h b/qemu/mips.h index 24b029f1..11f59a22 100644 --- a/qemu/mips.h +++ b/qemu/mips.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_mips #define address_space_read address_space_read_mips #define address_space_read_continue address_space_read_continue_mips +#define address_space_read_full address_space_read_full_mips #define address_space_rw address_space_rw_mips #define address_space_translate address_space_translate_mips #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips @@ -2040,7 +2041,6 @@ #define mem_add mem_add_mips #define mem_begin mem_begin_mips #define mem_commit mem_commit_mips -#define memory_access_is_direct memory_access_is_direct_mips #define memory_access_size memory_access_size_mips #define memory_init memory_init_mips #define memory_listener_match memory_listener_match_mips diff --git a/qemu/mips64.h b/qemu/mips64.h index d1c6452c..81f01afe 100644 --- a/qemu/mips64.h +++ b/qemu/mips64.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_mips64 #define address_space_read address_space_read_mips64 #define address_space_read_continue address_space_read_continue_mips64 +#define address_space_read_full address_space_read_full_mips64 #define address_space_rw address_space_rw_mips64 #define address_space_translate address_space_translate_mips64 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips64 @@ -2040,7 +2041,6 @@ #define mem_add mem_add_mips64 #define mem_begin mem_begin_mips64 #define mem_commit mem_commit_mips64 -#define memory_access_is_direct memory_access_is_direct_mips64 #define memory_access_size memory_access_size_mips64 #define memory_init memory_init_mips64 #define memory_listener_match memory_listener_match_mips64 diff --git a/qemu/mips64el.h b/qemu/mips64el.h index 0b1d24e4..bdc79ea2 100644 --- a/qemu/mips64el.h +++ b/qemu/mips64el.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_mips64el #define address_space_read address_space_read_mips64el #define address_space_read_continue address_space_read_continue_mips64el +#define address_space_read_full address_space_read_full_mips64el #define address_space_rw address_space_rw_mips64el #define address_space_translate address_space_translate_mips64el #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips64el @@ -2040,7 +2041,6 @@ #define mem_add mem_add_mips64el #define mem_begin mem_begin_mips64el #define mem_commit mem_commit_mips64el -#define memory_access_is_direct memory_access_is_direct_mips64el #define memory_access_size memory_access_size_mips64el #define memory_init memory_init_mips64el #define memory_listener_match memory_listener_match_mips64el diff --git a/qemu/mipsel.h b/qemu/mipsel.h index 1b112a06..c3ea2d86 100644 --- a/qemu/mipsel.h +++ b/qemu/mipsel.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_mipsel #define address_space_read address_space_read_mipsel #define address_space_read_continue address_space_read_continue_mipsel +#define address_space_read_full address_space_read_full_mipsel #define address_space_rw address_space_rw_mipsel #define address_space_translate address_space_translate_mipsel #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mipsel @@ -2040,7 +2041,6 @@ #define mem_add mem_add_mipsel #define mem_begin mem_begin_mipsel #define mem_commit mem_commit_mipsel -#define memory_access_is_direct memory_access_is_direct_mipsel #define memory_access_size memory_access_size_mipsel #define memory_init memory_init_mipsel #define memory_listener_match memory_listener_match_mipsel diff --git a/qemu/powerpc.h b/qemu/powerpc.h index be8da8ef..b2637eaf 100644 --- a/qemu/powerpc.h +++ b/qemu/powerpc.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_powerpc #define address_space_read address_space_read_powerpc #define address_space_read_continue address_space_read_continue_powerpc +#define address_space_read_full address_space_read_full_powerpc #define address_space_rw address_space_rw_powerpc #define address_space_translate address_space_translate_powerpc #define address_space_translate_for_iotlb address_space_translate_for_iotlb_powerpc @@ -2040,7 +2041,6 @@ #define mem_add mem_add_powerpc #define mem_begin mem_begin_powerpc #define mem_commit mem_commit_powerpc -#define memory_access_is_direct memory_access_is_direct_powerpc #define memory_access_size memory_access_size_powerpc #define memory_init memory_init_powerpc #define memory_listener_match memory_listener_match_powerpc diff --git a/qemu/sparc.h b/qemu/sparc.h index 38fccb18..9776b925 100644 --- a/qemu/sparc.h +++ b/qemu/sparc.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_sparc #define address_space_read address_space_read_sparc #define address_space_read_continue address_space_read_continue_sparc +#define address_space_read_full address_space_read_full_sparc #define address_space_rw address_space_rw_sparc #define address_space_translate address_space_translate_sparc #define address_space_translate_for_iotlb address_space_translate_for_iotlb_sparc @@ -2040,7 +2041,6 @@ #define mem_add mem_add_sparc #define mem_begin mem_begin_sparc #define mem_commit mem_commit_sparc -#define memory_access_is_direct memory_access_is_direct_sparc #define memory_access_size memory_access_size_sparc #define memory_init memory_init_sparc #define memory_listener_match memory_listener_match_sparc diff --git a/qemu/sparc64.h b/qemu/sparc64.h index 4f90d303..9f6b2eee 100644 --- a/qemu/sparc64.h +++ b/qemu/sparc64.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_sparc64 #define address_space_read address_space_read_sparc64 #define address_space_read_continue address_space_read_continue_sparc64 +#define address_space_read_full address_space_read_full_sparc64 #define address_space_rw address_space_rw_sparc64 #define address_space_translate address_space_translate_sparc64 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_sparc64 @@ -2040,7 +2041,6 @@ #define mem_add mem_add_sparc64 #define mem_begin mem_begin_sparc64 #define mem_commit mem_commit_sparc64 -#define memory_access_is_direct memory_access_is_direct_sparc64 #define memory_access_size memory_access_size_sparc64 #define memory_init memory_init_sparc64 #define memory_listener_match memory_listener_match_sparc64 diff --git a/qemu/x86_64.h b/qemu/x86_64.h index a10499cd..fa1c1753 100644 --- a/qemu/x86_64.h +++ b/qemu/x86_64.h @@ -94,6 +94,7 @@ #define address_space_map address_space_map_x86_64 #define address_space_read address_space_read_x86_64 #define address_space_read_continue address_space_read_continue_x86_64 +#define address_space_read_full address_space_read_full_x86_64 #define address_space_rw address_space_rw_x86_64 #define address_space_translate address_space_translate_x86_64 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_x86_64 @@ -2040,7 +2041,6 @@ #define mem_add mem_add_x86_64 #define mem_begin mem_begin_x86_64 #define mem_commit mem_commit_x86_64 -#define memory_access_is_direct memory_access_is_direct_x86_64 #define memory_access_size memory_access_size_x86_64 #define memory_init memory_init_x86_64 #define memory_listener_match memory_listener_match_x86_64