blob: 251dc79e10280145021a981890e4195a87393715 [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
Blue Swirl5b6dd862012-12-02 16:04:43 +00002 * Virtual page mapping
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
bellard67b915a2004-03-31 23:37:16 +000019#include "config.h"
Stefan Weil777872e2014-02-23 18:02:08 +010020#ifndef _WIN32
bellarda98d49b2004-11-14 16:22:05 +000021#include <sys/types.h>
bellardd5a8f072004-09-29 21:15:28 +000022#include <sys/mman.h>
23#endif
bellard54936002003-05-13 00:25:15 +000024
Stefan Weil055403b2010-10-22 23:03:32 +020025#include "qemu-common.h"
bellard6180a182003-09-30 21:04:53 +000026#include "cpu.h"
bellardb67d9a52008-05-23 09:57:34 +000027#include "tcg.h"
pbrookb3c77242008-06-30 16:31:04 +000028#include "hw/hw.h"
Michael S. Tsirkin4485bd22015-03-11 07:56:34 +010029#if !defined(CONFIG_USER_ONLY)
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +020030#include "hw/boards.h"
Michael S. Tsirkin4485bd22015-03-11 07:56:34 +010031#endif
Alex Williamsoncc9e98c2010-06-25 11:09:43 -060032#include "hw/qdev.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010033#include "qemu/osdep.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010034#include "sysemu/kvm.h"
Markus Armbruster2ff3de62013-07-04 15:09:22 +020035#include "sysemu/sysemu.h"
Paolo Bonzini0d09e412013-02-05 17:06:20 +010036#include "hw/xen/xen.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010037#include "qemu/timer.h"
38#include "qemu/config-file.h"
Andreas Färber75a34032013-09-02 16:57:02 +020039#include "qemu/error-report.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010040#include "exec/memory.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010041#include "sysemu/dma.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010042#include "exec/address-spaces.h"
pbrook53a59602006-03-25 19:31:22 +000043#if defined(CONFIG_USER_ONLY)
44#include <qemu.h>
Jun Nakajima432d2682010-08-31 16:41:25 +010045#else /* !CONFIG_USER_ONLY */
Paolo Bonzini9c17d612012-12-17 18:20:04 +010046#include "sysemu/xen-mapcache.h"
Stefano Stabellini6506e4f2011-05-19 18:35:44 +010047#include "trace.h"
pbrook53a59602006-03-25 19:31:22 +000048#endif
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +010049#include "exec/cpu-all.h"
Mike Day0dc3f442013-09-05 14:41:35 -040050#include "qemu/rcu_queue.h"
Jan Kiszka4840f102015-06-18 18:47:22 +020051#include "qemu/main-loop.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010052#include "exec/cputlb.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000053#include "translate-all.h"
Blue Swirl0cac1b62012-04-09 16:50:52 +000054
Paolo Bonzini022c62c2012-12-17 18:19:49 +010055#include "exec/memory-internal.h"
Juan Quintela220c3eb2013-10-14 17:13:59 +020056#include "exec/ram_addr.h"
Avi Kivity67d95c12011-12-15 15:25:22 +020057
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020058#include "qemu/range.h"
59
blueswir1db7b5422007-05-26 17:36:03 +000060//#define DEBUG_SUBPAGE
ths1196be32007-03-17 15:17:58 +000061
pbrook99773bd2006-04-16 15:14:59 +000062#if !defined(CONFIG_USER_ONLY)
Mike Day0dc3f442013-09-05 14:41:35 -040063/* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
64 * are protected by the ramlist lock.
65 */
Mike Day0d53d9f2015-01-21 13:45:24 +010066RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
Avi Kivity62152b82011-07-26 14:26:14 +030067
68static MemoryRegion *system_memory;
Avi Kivity309cb472011-08-08 16:09:03 +030069static MemoryRegion *system_io;
Avi Kivity62152b82011-07-26 14:26:14 +030070
Avi Kivityf6790af2012-10-02 20:13:51 +020071AddressSpace address_space_io;
72AddressSpace address_space_memory;
Avi Kivity2673a5d2012-10-02 18:49:28 +020073
Paolo Bonzini0844e002013-05-24 14:37:28 +020074MemoryRegion io_mem_rom, io_mem_notdirty;
Jan Kiszkaacc9d802013-05-26 21:55:37 +020075static MemoryRegion io_mem_unassigned;
Avi Kivity0e0df1e2012-01-02 00:32:15 +020076
Paolo Bonzini7bd4f432014-05-14 17:43:22 +080077/* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
78#define RAM_PREALLOC (1 << 0)
79
Paolo Bonzinidbcb8982014-06-10 19:15:24 +080080/* RAM is mmap-ed with MAP_SHARED */
81#define RAM_SHARED (1 << 1)
82
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +020083/* Only a portion of RAM (used_length) is actually used, and migrated.
84 * This used_length size can change across reboots.
85 */
86#define RAM_RESIZEABLE (1 << 2)
87
pbrooke2eef172008-06-08 01:09:01 +000088#endif
bellard9fa3e852004-01-04 18:06:42 +000089
Andreas Färberbdc44642013-06-24 23:50:24 +020090struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
bellard6a00d602005-11-21 23:25:50 +000091/* current CPU in the current thread. It is only valid inside
92 cpu_exec() */
Andreas Färber4917cf42013-05-27 05:17:50 +020093DEFINE_TLS(CPUState *, current_cpu);
pbrook2e70f6e2008-06-29 01:03:05 +000094/* 0 = Do not count executed instructions.
thsbf20dc02008-06-30 17:22:19 +000095 1 = Precise instruction counting.
pbrook2e70f6e2008-06-29 01:03:05 +000096 2 = Adaptive rate instruction counting. */
Paolo Bonzini5708fc62012-11-26 15:36:40 +010097int use_icount;
bellard6a00d602005-11-21 23:25:50 +000098
pbrooke2eef172008-06-08 01:09:01 +000099#if !defined(CONFIG_USER_ONLY)
Avi Kivity4346ae32012-02-10 17:00:01 +0200100
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200101typedef struct PhysPageEntry PhysPageEntry;
102
103struct PhysPageEntry {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200104 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200105 uint32_t skip : 6;
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200106 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200107 uint32_t ptr : 26;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200108};
109
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200110#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
111
Paolo Bonzini03f49952013-11-07 17:14:36 +0100112/* Size of the L2 (and L3, etc) page tables. */
Paolo Bonzini57271d62013-11-07 17:14:37 +0100113#define ADDR_SPACE_BITS 64
Paolo Bonzini03f49952013-11-07 17:14:36 +0100114
Michael S. Tsirkin026736c2013-11-13 20:13:03 +0200115#define P_L2_BITS 9
Paolo Bonzini03f49952013-11-07 17:14:36 +0100116#define P_L2_SIZE (1 << P_L2_BITS)
117
118#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
119
120typedef PhysPageEntry Node[P_L2_SIZE];
Paolo Bonzini0475d942013-05-29 12:28:21 +0200121
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200122typedef struct PhysPageMap {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100123 struct rcu_head rcu;
124
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200125 unsigned sections_nb;
126 unsigned sections_nb_alloc;
127 unsigned nodes_nb;
128 unsigned nodes_nb_alloc;
129 Node *nodes;
130 MemoryRegionSection *sections;
131} PhysPageMap;
132
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200133struct AddressSpaceDispatch {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100134 struct rcu_head rcu;
135
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200136 /* This is a multi-level map on the physical address space.
137 * The bottom level has pointers to MemoryRegionSections.
138 */
139 PhysPageEntry phys_map;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200140 PhysPageMap map;
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200141 AddressSpace *as;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200142};
143
Jan Kiszka90260c62013-05-26 21:46:51 +0200144#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
145typedef struct subpage_t {
146 MemoryRegion iomem;
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200147 AddressSpace *as;
Jan Kiszka90260c62013-05-26 21:46:51 +0200148 hwaddr base;
149 uint16_t sub_section[TARGET_PAGE_SIZE];
150} subpage_t;
151
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200152#define PHYS_SECTION_UNASSIGNED 0
153#define PHYS_SECTION_NOTDIRTY 1
154#define PHYS_SECTION_ROM 2
155#define PHYS_SECTION_WATCH 3
Avi Kivity5312bd82012-02-12 18:32:55 +0200156
pbrooke2eef172008-06-08 01:09:01 +0000157static void io_mem_init(void);
Avi Kivity62152b82011-07-26 14:26:14 +0300158static void memory_map_init(void);
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000159static void tcg_commit(MemoryListener *listener);
pbrooke2eef172008-06-08 01:09:01 +0000160
Avi Kivity1ec9b902012-01-02 12:47:48 +0200161static MemoryRegion io_mem_watch;
pbrook6658ffb2007-03-16 23:58:11 +0000162#endif
bellard54936002003-05-13 00:25:15 +0000163
Paul Brook6d9a1302010-02-28 23:55:53 +0000164#if !defined(CONFIG_USER_ONLY)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200165
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200166static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200167{
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200168 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
169 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
170 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
171 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200172 }
173}
174
Paolo Bonzinidb946042015-05-21 15:12:29 +0200175static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200176{
177 unsigned i;
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200178 uint32_t ret;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200179 PhysPageEntry e;
180 PhysPageEntry *p;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200181
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200182 ret = map->nodes_nb++;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200183 p = map->nodes[ret];
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200184 assert(ret != PHYS_MAP_NODE_NIL);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200185 assert(ret != map->nodes_nb_alloc);
Paolo Bonzinidb946042015-05-21 15:12:29 +0200186
187 e.skip = leaf ? 0 : 1;
188 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100189 for (i = 0; i < P_L2_SIZE; ++i) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200190 memcpy(&p[i], &e, sizeof(e));
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200191 }
Avi Kivityf7bf5462012-02-13 20:12:05 +0200192 return ret;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200193}
194
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200195static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
196 hwaddr *index, hwaddr *nb, uint16_t leaf,
Avi Kivity29990972012-02-13 20:21:20 +0200197 int level)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200198{
199 PhysPageEntry *p;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100200 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200201
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200202 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200203 lp->ptr = phys_map_node_alloc(map, level == 0);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200204 }
Paolo Bonzinidb946042015-05-21 15:12:29 +0200205 p = map->nodes[lp->ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100206 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf7bf5462012-02-13 20:12:05 +0200207
Paolo Bonzini03f49952013-11-07 17:14:36 +0100208 while (*nb && lp < &p[P_L2_SIZE]) {
Avi Kivity07f07b32012-02-13 20:45:32 +0200209 if ((*index & (step - 1)) == 0 && *nb >= step) {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200210 lp->skip = 0;
Avi Kivityc19e8802012-02-13 20:25:31 +0200211 lp->ptr = leaf;
Avi Kivity07f07b32012-02-13 20:45:32 +0200212 *index += step;
213 *nb -= step;
Avi Kivity29990972012-02-13 20:21:20 +0200214 } else {
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200215 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
Avi Kivity29990972012-02-13 20:21:20 +0200216 }
217 ++lp;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200218 }
219}
220
Avi Kivityac1970f2012-10-03 16:22:53 +0200221static void phys_page_set(AddressSpaceDispatch *d,
Avi Kivitya8170e52012-10-23 12:30:10 +0200222 hwaddr index, hwaddr nb,
Avi Kivity29990972012-02-13 20:21:20 +0200223 uint16_t leaf)
bellard92e873b2004-05-21 14:52:29 +0000224{
Avi Kivity29990972012-02-13 20:21:20 +0200225 /* Wildly overreserve - it doesn't matter much. */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200226 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
bellard92e873b2004-05-21 14:52:29 +0000227
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200228 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
bellard92e873b2004-05-21 14:52:29 +0000229}
230
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200231/* Compact a non leaf page entry. Simply detect that the entry has a single child,
232 * and update our entry so we can skip it and go directly to the destination.
233 */
234static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
235{
236 unsigned valid_ptr = P_L2_SIZE;
237 int valid = 0;
238 PhysPageEntry *p;
239 int i;
240
241 if (lp->ptr == PHYS_MAP_NODE_NIL) {
242 return;
243 }
244
245 p = nodes[lp->ptr];
246 for (i = 0; i < P_L2_SIZE; i++) {
247 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
248 continue;
249 }
250
251 valid_ptr = i;
252 valid++;
253 if (p[i].skip) {
254 phys_page_compact(&p[i], nodes, compacted);
255 }
256 }
257
258 /* We can only compress if there's only one child. */
259 if (valid != 1) {
260 return;
261 }
262
263 assert(valid_ptr < P_L2_SIZE);
264
265 /* Don't compress if it won't fit in the # of bits we have. */
266 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
267 return;
268 }
269
270 lp->ptr = p[valid_ptr].ptr;
271 if (!p[valid_ptr].skip) {
272 /* If our only child is a leaf, make this a leaf. */
273 /* By design, we should have made this node a leaf to begin with so we
274 * should never reach here.
275 * But since it's so simple to handle this, let's do it just in case we
276 * change this rule.
277 */
278 lp->skip = 0;
279 } else {
280 lp->skip += p[valid_ptr].skip;
281 }
282}
283
284static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
285{
286 DECLARE_BITMAP(compacted, nodes_nb);
287
288 if (d->phys_map.skip) {
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200289 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200290 }
291}
292
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200293static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200294 Node *nodes, MemoryRegionSection *sections)
bellard92e873b2004-05-21 14:52:29 +0000295{
Avi Kivity31ab2b42012-02-13 16:44:19 +0200296 PhysPageEntry *p;
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200297 hwaddr index = addr >> TARGET_PAGE_BITS;
Avi Kivity31ab2b42012-02-13 16:44:19 +0200298 int i;
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200299
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200300 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
Avi Kivityc19e8802012-02-13 20:25:31 +0200301 if (lp.ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200302 return &sections[PHYS_SECTION_UNASSIGNED];
Avi Kivity31ab2b42012-02-13 16:44:19 +0200303 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200304 p = nodes[lp.ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100305 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200306 }
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200307
308 if (sections[lp.ptr].size.hi ||
309 range_covers_byte(sections[lp.ptr].offset_within_address_space,
310 sections[lp.ptr].size.lo, addr)) {
311 return &sections[lp.ptr];
312 } else {
313 return &sections[PHYS_SECTION_UNASSIGNED];
314 }
Avi Kivityf3705d52012-03-08 16:16:34 +0200315}
316
Blue Swirle5548612012-04-21 13:08:33 +0000317bool memory_region_is_unassigned(MemoryRegion *mr)
318{
Paolo Bonzini2a8e7492013-05-24 14:34:08 +0200319 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
Blue Swirle5548612012-04-21 13:08:33 +0000320 && mr != &io_mem_watch;
321}
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200322
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100323/* Called from RCU critical section */
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200324static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
Jan Kiszka90260c62013-05-26 21:46:51 +0200325 hwaddr addr,
326 bool resolve_subpage)
Jan Kiszka9f029602013-05-06 16:48:02 +0200327{
Jan Kiszka90260c62013-05-26 21:46:51 +0200328 MemoryRegionSection *section;
329 subpage_t *subpage;
330
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200331 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
Jan Kiszka90260c62013-05-26 21:46:51 +0200332 if (resolve_subpage && section->mr->subpage) {
333 subpage = container_of(section->mr, subpage_t, iomem);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200334 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
Jan Kiszka90260c62013-05-26 21:46:51 +0200335 }
336 return section;
Jan Kiszka9f029602013-05-06 16:48:02 +0200337}
338
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100339/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200340static MemoryRegionSection *
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200341address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
Jan Kiszka90260c62013-05-26 21:46:51 +0200342 hwaddr *plen, bool resolve_subpage)
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200343{
344 MemoryRegionSection *section;
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200345 MemoryRegion *mr;
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100346 Int128 diff;
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200347
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200348 section = address_space_lookup_region(d, addr, resolve_subpage);
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200349 /* Compute offset within MemoryRegionSection */
350 addr -= section->offset_within_address_space;
351
352 /* Compute offset within MemoryRegion */
353 *xlat = addr + section->offset_within_region;
354
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200355 mr = section->mr;
Paolo Bonzinib242e0e2015-07-04 00:24:51 +0200356
357 /* MMIO registers can be expected to perform full-width accesses based only
358 * on their address, without considering adjacent registers that could
359 * decode to completely different MemoryRegions. When such registers
360 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
361 * regions overlap wildly. For this reason we cannot clamp the accesses
362 * here.
363 *
364 * If the length is small (as is the case for address_space_ldl/stl),
365 * everything works fine. If the incoming length is large, however,
366 * the caller really has to do the clamping through memory_access_size.
367 */
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200368 if (memory_region_is_ram(mr)) {
Paolo Bonzinie4a511f2015-06-17 10:36:54 +0200369 diff = int128_sub(section->size, int128_make64(addr));
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200370 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
371 }
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200372 return section;
373}
Jan Kiszka90260c62013-05-26 21:46:51 +0200374
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100375static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
376{
377 if (memory_region_is_ram(mr)) {
378 return !(is_write && mr->readonly);
379 }
380 if (memory_region_is_romd(mr)) {
381 return !is_write;
382 }
383
384 return false;
385}
386
Paolo Bonzini41063e12015-03-18 14:21:43 +0100387/* Called from RCU critical section */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +0200388MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
389 hwaddr *xlat, hwaddr *plen,
390 bool is_write)
Jan Kiszka90260c62013-05-26 21:46:51 +0200391{
Avi Kivity30951152012-10-30 13:47:46 +0200392 IOMMUTLBEntry iotlb;
393 MemoryRegionSection *section;
394 MemoryRegion *mr;
Avi Kivity30951152012-10-30 13:47:46 +0200395
396 for (;;) {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100397 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
398 section = address_space_translate_internal(d, addr, &addr, plen, true);
Avi Kivity30951152012-10-30 13:47:46 +0200399 mr = section->mr;
400
401 if (!mr->iommu_ops) {
402 break;
403 }
404
Le Tan8d7b8cb2014-08-16 13:55:37 +0800405 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
Avi Kivity30951152012-10-30 13:47:46 +0200406 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
407 | (addr & iotlb.addr_mask));
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700408 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
Avi Kivity30951152012-10-30 13:47:46 +0200409 if (!(iotlb.perm & (1 << is_write))) {
410 mr = &io_mem_unassigned;
411 break;
412 }
413
414 as = iotlb.target_as;
415 }
416
Alexey Kardashevskiyfe680d02014-05-07 13:40:39 +0000417 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100418 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700419 *plen = MIN(page, *plen);
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100420 }
421
Avi Kivity30951152012-10-30 13:47:46 +0200422 *xlat = addr;
423 return mr;
Jan Kiszka90260c62013-05-26 21:46:51 +0200424}
425
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100426/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200427MemoryRegionSection *
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +0200428address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
429 hwaddr *xlat, hwaddr *plen)
Jan Kiszka90260c62013-05-26 21:46:51 +0200430{
Avi Kivity30951152012-10-30 13:47:46 +0200431 MemoryRegionSection *section;
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +0200432 section = address_space_translate_internal(cpu->memory_dispatch,
433 addr, xlat, plen, false);
Avi Kivity30951152012-10-30 13:47:46 +0200434
435 assert(!section->mr->iommu_ops);
436 return section;
Jan Kiszka90260c62013-05-26 21:46:51 +0200437}
bellard9fa3e852004-01-04 18:06:42 +0000438#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000439
Andreas Färberb170fce2013-01-20 20:23:22 +0100440#if !defined(CONFIG_USER_ONLY)
pbrook9656f322008-07-01 20:01:19 +0000441
Juan Quintelae59fb372009-09-29 22:48:21 +0200442static int cpu_common_post_load(void *opaque, int version_id)
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200443{
Andreas Färber259186a2013-01-17 18:51:17 +0100444 CPUState *cpu = opaque;
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200445
aurel323098dba2009-03-07 21:28:24 +0000446 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
447 version_id is increased. */
Andreas Färber259186a2013-01-17 18:51:17 +0100448 cpu->interrupt_request &= ~0x01;
Christian Borntraegerc01a71c2014-03-17 17:13:12 +0100449 tlb_flush(cpu, 1);
pbrook9656f322008-07-01 20:01:19 +0000450
451 return 0;
452}
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200453
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400454static int cpu_common_pre_load(void *opaque)
455{
456 CPUState *cpu = opaque;
457
Paolo Bonziniadee6422014-12-19 12:53:14 +0100458 cpu->exception_index = -1;
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400459
460 return 0;
461}
462
463static bool cpu_common_exception_index_needed(void *opaque)
464{
465 CPUState *cpu = opaque;
466
Paolo Bonziniadee6422014-12-19 12:53:14 +0100467 return tcg_enabled() && cpu->exception_index != -1;
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400468}
469
470static const VMStateDescription vmstate_cpu_common_exception_index = {
471 .name = "cpu_common/exception_index",
472 .version_id = 1,
473 .minimum_version_id = 1,
Juan Quintela5cd8cad2014-09-23 14:09:54 +0200474 .needed = cpu_common_exception_index_needed,
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400475 .fields = (VMStateField[]) {
476 VMSTATE_INT32(exception_index, CPUState),
477 VMSTATE_END_OF_LIST()
478 }
479};
480
Andreas Färber1a1562f2013-06-17 04:09:11 +0200481const VMStateDescription vmstate_cpu_common = {
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200482 .name = "cpu_common",
483 .version_id = 1,
484 .minimum_version_id = 1,
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400485 .pre_load = cpu_common_pre_load,
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200486 .post_load = cpu_common_post_load,
Juan Quintela35d08452014-04-16 16:01:33 +0200487 .fields = (VMStateField[]) {
Andreas Färber259186a2013-01-17 18:51:17 +0100488 VMSTATE_UINT32(halted, CPUState),
489 VMSTATE_UINT32(interrupt_request, CPUState),
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200490 VMSTATE_END_OF_LIST()
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400491 },
Juan Quintela5cd8cad2014-09-23 14:09:54 +0200492 .subsections = (const VMStateDescription*[]) {
493 &vmstate_cpu_common_exception_index,
494 NULL
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200495 }
496};
Andreas Färber1a1562f2013-06-17 04:09:11 +0200497
pbrook9656f322008-07-01 20:01:19 +0000498#endif
499
Andreas Färber38d8f5c2012-12-17 19:47:15 +0100500CPUState *qemu_get_cpu(int index)
Glauber Costa950f1472009-06-09 12:15:18 -0400501{
Andreas Färberbdc44642013-06-24 23:50:24 +0200502 CPUState *cpu;
Glauber Costa950f1472009-06-09 12:15:18 -0400503
Andreas Färberbdc44642013-06-24 23:50:24 +0200504 CPU_FOREACH(cpu) {
Andreas Färber55e5c282012-12-17 06:18:02 +0100505 if (cpu->cpu_index == index) {
Andreas Färberbdc44642013-06-24 23:50:24 +0200506 return cpu;
Andreas Färber55e5c282012-12-17 06:18:02 +0100507 }
Glauber Costa950f1472009-06-09 12:15:18 -0400508 }
509
Andreas Färberbdc44642013-06-24 23:50:24 +0200510 return NULL;
Glauber Costa950f1472009-06-09 12:15:18 -0400511}
512
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000513#if !defined(CONFIG_USER_ONLY)
514void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
515{
516 /* We only support one address space per cpu at the moment. */
517 assert(cpu->as == as);
518
519 if (cpu->tcg_as_listener) {
520 memory_listener_unregister(cpu->tcg_as_listener);
521 } else {
522 cpu->tcg_as_listener = g_new0(MemoryListener, 1);
523 }
524 cpu->tcg_as_listener->commit = tcg_commit;
525 memory_listener_register(cpu->tcg_as_listener, as);
526}
527#endif
528
Andreas Färber9349b4f2012-03-14 01:38:32 +0100529void cpu_exec_init(CPUArchState *env)
bellardfd6ce8f2003-05-14 19:00:11 +0000530{
Andreas Färber9f09e182012-05-03 06:59:07 +0200531 CPUState *cpu = ENV_GET_CPU(env);
Andreas Färberb170fce2013-01-20 20:23:22 +0100532 CPUClass *cc = CPU_GET_CLASS(cpu);
Andreas Färberbdc44642013-06-24 23:50:24 +0200533 CPUState *some_cpu;
bellard6a00d602005-11-21 23:25:50 +0000534 int cpu_index;
535
pbrookc2764712009-03-07 15:24:59 +0000536#if defined(CONFIG_USER_ONLY)
537 cpu_list_lock();
538#endif
bellard6a00d602005-11-21 23:25:50 +0000539 cpu_index = 0;
Andreas Färberbdc44642013-06-24 23:50:24 +0200540 CPU_FOREACH(some_cpu) {
bellard6a00d602005-11-21 23:25:50 +0000541 cpu_index++;
542 }
Andreas Färber55e5c282012-12-17 06:18:02 +0100543 cpu->cpu_index = cpu_index;
Andreas Färber1b1ed8d2012-12-17 04:22:03 +0100544 cpu->numa_node = 0;
Andreas Färberf0c3c502013-08-26 21:22:53 +0200545 QTAILQ_INIT(&cpu->breakpoints);
Andreas Färberff4700b2013-08-26 18:23:18 +0200546 QTAILQ_INIT(&cpu->watchpoints);
Jan Kiszkadc7a09c2011-03-15 12:26:31 +0100547#ifndef CONFIG_USER_ONLY
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000548 cpu->as = &address_space_memory;
Andreas Färber9f09e182012-05-03 06:59:07 +0200549 cpu->thread_id = qemu_get_thread_id();
Paolo Bonzinicba70542015-03-09 15:28:37 +0100550 cpu_reload_memory_map(cpu);
Jan Kiszkadc7a09c2011-03-15 12:26:31 +0100551#endif
Andreas Färberbdc44642013-06-24 23:50:24 +0200552 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
pbrookc2764712009-03-07 15:24:59 +0000553#if defined(CONFIG_USER_ONLY)
554 cpu_list_unlock();
555#endif
Andreas Färbere0d47942013-07-29 04:07:50 +0200556 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
557 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
558 }
pbrookb3c77242008-06-30 16:31:04 +0000559#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
Alex Williamson0be71e32010-06-25 11:09:07 -0600560 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
pbrookb3c77242008-06-30 16:31:04 +0000561 cpu_save, cpu_load, env);
Andreas Färberb170fce2013-01-20 20:23:22 +0100562 assert(cc->vmsd == NULL);
Andreas Färbere0d47942013-07-29 04:07:50 +0200563 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
pbrookb3c77242008-06-30 16:31:04 +0000564#endif
Andreas Färberb170fce2013-01-20 20:23:22 +0100565 if (cc->vmsd != NULL) {
566 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
567 }
bellardfd6ce8f2003-05-14 19:00:11 +0000568}
569
Paul Brook94df27f2010-02-28 23:47:45 +0000570#if defined(CONFIG_USER_ONLY)
Andreas Färber00b941e2013-06-29 18:55:54 +0200571static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
Paul Brook94df27f2010-02-28 23:47:45 +0000572{
573 tb_invalidate_phys_page_range(pc, pc + 1, 0);
574}
575#else
Andreas Färber00b941e2013-06-29 18:55:54 +0200576static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
Max Filippov1e7855a2012-04-10 02:48:17 +0400577{
Max Filippove8262a12013-09-27 22:29:17 +0400578 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
579 if (phys != -1) {
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000580 tb_invalidate_phys_addr(cpu->as,
Edgar E. Iglesias29d8ec72013-11-07 19:43:10 +0100581 phys | (pc & ~TARGET_PAGE_MASK));
Max Filippove8262a12013-09-27 22:29:17 +0400582 }
Max Filippov1e7855a2012-04-10 02:48:17 +0400583}
bellardc27004e2005-01-03 23:35:10 +0000584#endif
bellardd720b932004-04-25 17:57:43 +0000585
Paul Brookc527ee82010-03-01 03:31:14 +0000586#if defined(CONFIG_USER_ONLY)
Andreas Färber75a34032013-09-02 16:57:02 +0200587void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
Paul Brookc527ee82010-03-01 03:31:14 +0000588
589{
590}
591
Peter Maydell3ee887e2014-09-12 14:06:48 +0100592int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
593 int flags)
594{
595 return -ENOSYS;
596}
597
598void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
599{
600}
601
Andreas Färber75a34032013-09-02 16:57:02 +0200602int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
Paul Brookc527ee82010-03-01 03:31:14 +0000603 int flags, CPUWatchpoint **watchpoint)
604{
605 return -ENOSYS;
606}
607#else
pbrook6658ffb2007-03-16 23:58:11 +0000608/* Add a watchpoint. */
Andreas Färber75a34032013-09-02 16:57:02 +0200609int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
aliguoria1d1bb32008-11-18 20:07:32 +0000610 int flags, CPUWatchpoint **watchpoint)
pbrook6658ffb2007-03-16 23:58:11 +0000611{
aliguoric0ce9982008-11-25 22:13:57 +0000612 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +0000613
Peter Maydell05068c02014-09-12 14:06:48 +0100614 /* forbid ranges which are empty or run off the end of the address space */
Max Filippov07e28632014-09-17 22:03:36 -0700615 if (len == 0 || (addr + len - 1) < addr) {
Andreas Färber75a34032013-09-02 16:57:02 +0200616 error_report("tried to set invalid watchpoint at %"
617 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
aliguorib4051332008-11-18 20:14:20 +0000618 return -EINVAL;
619 }
Anthony Liguori7267c092011-08-20 22:09:37 -0500620 wp = g_malloc(sizeof(*wp));
pbrook6658ffb2007-03-16 23:58:11 +0000621
aliguoria1d1bb32008-11-18 20:07:32 +0000622 wp->vaddr = addr;
Peter Maydell05068c02014-09-12 14:06:48 +0100623 wp->len = len;
aliguoria1d1bb32008-11-18 20:07:32 +0000624 wp->flags = flags;
625
aliguori2dc9f412008-11-18 20:56:59 +0000626 /* keep all GDB-injected watchpoints in front */
Andreas Färberff4700b2013-08-26 18:23:18 +0200627 if (flags & BP_GDB) {
628 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
629 } else {
630 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
631 }
aliguoria1d1bb32008-11-18 20:07:32 +0000632
Andreas Färber31b030d2013-09-04 01:29:02 +0200633 tlb_flush_page(cpu, addr);
aliguoria1d1bb32008-11-18 20:07:32 +0000634
635 if (watchpoint)
636 *watchpoint = wp;
637 return 0;
pbrook6658ffb2007-03-16 23:58:11 +0000638}
639
aliguoria1d1bb32008-11-18 20:07:32 +0000640/* Remove a specific watchpoint. */
Andreas Färber75a34032013-09-02 16:57:02 +0200641int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
aliguoria1d1bb32008-11-18 20:07:32 +0000642 int flags)
pbrook6658ffb2007-03-16 23:58:11 +0000643{
aliguoria1d1bb32008-11-18 20:07:32 +0000644 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +0000645
Andreas Färberff4700b2013-08-26 18:23:18 +0200646 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +0100647 if (addr == wp->vaddr && len == wp->len
aliguori6e140f22008-11-18 20:37:55 +0000648 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
Andreas Färber75a34032013-09-02 16:57:02 +0200649 cpu_watchpoint_remove_by_ref(cpu, wp);
pbrook6658ffb2007-03-16 23:58:11 +0000650 return 0;
651 }
652 }
aliguoria1d1bb32008-11-18 20:07:32 +0000653 return -ENOENT;
pbrook6658ffb2007-03-16 23:58:11 +0000654}
655
aliguoria1d1bb32008-11-18 20:07:32 +0000656/* Remove a specific watchpoint by reference. */
Andreas Färber75a34032013-09-02 16:57:02 +0200657void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
aliguoria1d1bb32008-11-18 20:07:32 +0000658{
Andreas Färberff4700b2013-08-26 18:23:18 +0200659 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
edgar_igl7d03f822008-05-17 18:58:29 +0000660
Andreas Färber31b030d2013-09-04 01:29:02 +0200661 tlb_flush_page(cpu, watchpoint->vaddr);
aliguoria1d1bb32008-11-18 20:07:32 +0000662
Anthony Liguori7267c092011-08-20 22:09:37 -0500663 g_free(watchpoint);
edgar_igl7d03f822008-05-17 18:58:29 +0000664}
665
aliguoria1d1bb32008-11-18 20:07:32 +0000666/* Remove all matching watchpoints. */
Andreas Färber75a34032013-09-02 16:57:02 +0200667void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
aliguoria1d1bb32008-11-18 20:07:32 +0000668{
aliguoric0ce9982008-11-25 22:13:57 +0000669 CPUWatchpoint *wp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +0000670
Andreas Färberff4700b2013-08-26 18:23:18 +0200671 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
Andreas Färber75a34032013-09-02 16:57:02 +0200672 if (wp->flags & mask) {
673 cpu_watchpoint_remove_by_ref(cpu, wp);
674 }
aliguoric0ce9982008-11-25 22:13:57 +0000675 }
aliguoria1d1bb32008-11-18 20:07:32 +0000676}
Peter Maydell05068c02014-09-12 14:06:48 +0100677
678/* Return true if this watchpoint address matches the specified
679 * access (ie the address range covered by the watchpoint overlaps
680 * partially or completely with the address range covered by the
681 * access).
682 */
683static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
684 vaddr addr,
685 vaddr len)
686{
687 /* We know the lengths are non-zero, but a little caution is
688 * required to avoid errors in the case where the range ends
689 * exactly at the top of the address space and so addr + len
690 * wraps round to zero.
691 */
692 vaddr wpend = wp->vaddr + wp->len - 1;
693 vaddr addrend = addr + len - 1;
694
695 return !(addr > wpend || wp->vaddr > addrend);
696}
697
Paul Brookc527ee82010-03-01 03:31:14 +0000698#endif
aliguoria1d1bb32008-11-18 20:07:32 +0000699
700/* Add a breakpoint. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200701int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
aliguoria1d1bb32008-11-18 20:07:32 +0000702 CPUBreakpoint **breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +0000703{
aliguoric0ce9982008-11-25 22:13:57 +0000704 CPUBreakpoint *bp;
ths3b46e622007-09-17 08:09:54 +0000705
Anthony Liguori7267c092011-08-20 22:09:37 -0500706 bp = g_malloc(sizeof(*bp));
aliguoria1d1bb32008-11-18 20:07:32 +0000707
708 bp->pc = pc;
709 bp->flags = flags;
710
aliguori2dc9f412008-11-18 20:56:59 +0000711 /* keep all GDB-injected breakpoints in front */
Andreas Färber00b941e2013-06-29 18:55:54 +0200712 if (flags & BP_GDB) {
Andreas Färberf0c3c502013-08-26 21:22:53 +0200713 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
Andreas Färber00b941e2013-06-29 18:55:54 +0200714 } else {
Andreas Färberf0c3c502013-08-26 21:22:53 +0200715 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
Andreas Färber00b941e2013-06-29 18:55:54 +0200716 }
aliguoria1d1bb32008-11-18 20:07:32 +0000717
Andreas Färberf0c3c502013-08-26 21:22:53 +0200718 breakpoint_invalidate(cpu, pc);
aliguoria1d1bb32008-11-18 20:07:32 +0000719
Andreas Färber00b941e2013-06-29 18:55:54 +0200720 if (breakpoint) {
aliguoria1d1bb32008-11-18 20:07:32 +0000721 *breakpoint = bp;
Andreas Färber00b941e2013-06-29 18:55:54 +0200722 }
aliguoria1d1bb32008-11-18 20:07:32 +0000723 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +0000724}
725
726/* Remove a specific breakpoint. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200727int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
aliguoria1d1bb32008-11-18 20:07:32 +0000728{
aliguoria1d1bb32008-11-18 20:07:32 +0000729 CPUBreakpoint *bp;
730
Andreas Färberf0c3c502013-08-26 21:22:53 +0200731 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
aliguoria1d1bb32008-11-18 20:07:32 +0000732 if (bp->pc == pc && bp->flags == flags) {
Andreas Färberb3310ab2013-09-02 17:26:20 +0200733 cpu_breakpoint_remove_by_ref(cpu, bp);
bellard4c3a88a2003-07-26 12:06:08 +0000734 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +0000735 }
bellard4c3a88a2003-07-26 12:06:08 +0000736 }
aliguoria1d1bb32008-11-18 20:07:32 +0000737 return -ENOENT;
bellard4c3a88a2003-07-26 12:06:08 +0000738}
739
aliguoria1d1bb32008-11-18 20:07:32 +0000740/* Remove a specific breakpoint by reference. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200741void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +0000742{
Andreas Färberf0c3c502013-08-26 21:22:53 +0200743 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
744
745 breakpoint_invalidate(cpu, breakpoint->pc);
aliguoria1d1bb32008-11-18 20:07:32 +0000746
Anthony Liguori7267c092011-08-20 22:09:37 -0500747 g_free(breakpoint);
aliguoria1d1bb32008-11-18 20:07:32 +0000748}
749
750/* Remove all matching breakpoints. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200751void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
aliguoria1d1bb32008-11-18 20:07:32 +0000752{
aliguoric0ce9982008-11-25 22:13:57 +0000753 CPUBreakpoint *bp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +0000754
Andreas Färberf0c3c502013-08-26 21:22:53 +0200755 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
Andreas Färberb3310ab2013-09-02 17:26:20 +0200756 if (bp->flags & mask) {
757 cpu_breakpoint_remove_by_ref(cpu, bp);
758 }
aliguoric0ce9982008-11-25 22:13:57 +0000759 }
bellard4c3a88a2003-07-26 12:06:08 +0000760}
761
bellardc33a3462003-07-29 20:50:33 +0000762/* enable or disable single step mode. EXCP_DEBUG is returned by the
763 CPU loop after each instruction */
Andreas Färber3825b282013-06-24 18:41:06 +0200764void cpu_single_step(CPUState *cpu, int enabled)
bellardc33a3462003-07-29 20:50:33 +0000765{
Andreas Färbered2803d2013-06-21 20:20:45 +0200766 if (cpu->singlestep_enabled != enabled) {
767 cpu->singlestep_enabled = enabled;
768 if (kvm_enabled()) {
Stefan Weil38e478e2013-07-25 20:50:21 +0200769 kvm_update_guest_debug(cpu, 0);
Andreas Färbered2803d2013-06-21 20:20:45 +0200770 } else {
Stuart Bradyccbb4d42009-05-03 12:15:06 +0100771 /* must flush all the translated code to avoid inconsistencies */
aliguorie22a25c2009-03-12 20:12:48 +0000772 /* XXX: only flush what is necessary */
Stefan Weil38e478e2013-07-25 20:50:21 +0200773 CPUArchState *env = cpu->env_ptr;
aliguorie22a25c2009-03-12 20:12:48 +0000774 tb_flush(env);
775 }
bellardc33a3462003-07-29 20:50:33 +0000776 }
bellardc33a3462003-07-29 20:50:33 +0000777}
778
Andreas Färbera47dddd2013-09-03 17:38:47 +0200779void cpu_abort(CPUState *cpu, const char *fmt, ...)
bellard75012672003-06-21 13:11:07 +0000780{
781 va_list ap;
pbrook493ae1f2007-11-23 16:53:59 +0000782 va_list ap2;
bellard75012672003-06-21 13:11:07 +0000783
784 va_start(ap, fmt);
pbrook493ae1f2007-11-23 16:53:59 +0000785 va_copy(ap2, ap);
bellard75012672003-06-21 13:11:07 +0000786 fprintf(stderr, "qemu: fatal: ");
787 vfprintf(stderr, fmt, ap);
788 fprintf(stderr, "\n");
Andreas Färber878096e2013-05-27 01:33:50 +0200789 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
aliguori93fcfe32009-01-15 22:34:14 +0000790 if (qemu_log_enabled()) {
791 qemu_log("qemu: fatal: ");
792 qemu_log_vprintf(fmt, ap2);
793 qemu_log("\n");
Andreas Färbera0762852013-06-16 07:28:50 +0200794 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
aliguori31b1a7b2009-01-15 22:35:09 +0000795 qemu_log_flush();
aliguori93fcfe32009-01-15 22:34:14 +0000796 qemu_log_close();
balrog924edca2007-06-10 14:07:13 +0000797 }
pbrook493ae1f2007-11-23 16:53:59 +0000798 va_end(ap2);
j_mayerf9373292007-09-29 12:18:20 +0000799 va_end(ap);
Riku Voipiofd052bf2010-01-25 14:30:49 +0200800#if defined(CONFIG_USER_ONLY)
801 {
802 struct sigaction act;
803 sigfillset(&act.sa_mask);
804 act.sa_handler = SIG_DFL;
805 sigaction(SIGABRT, &act, NULL);
806 }
807#endif
bellard75012672003-06-21 13:11:07 +0000808 abort();
809}
810
bellard01243112004-01-04 15:48:17 +0000811#if !defined(CONFIG_USER_ONLY)
Mike Day0dc3f442013-09-05 14:41:35 -0400812/* Called from RCU critical section */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200813static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
814{
815 RAMBlock *block;
816
Paolo Bonzini43771532013-09-09 17:58:40 +0200817 block = atomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200818 if (block && addr - block->offset < block->max_length) {
Paolo Bonzini041603f2013-09-09 17:49:45 +0200819 goto found;
820 }
Mike Day0dc3f442013-09-05 14:41:35 -0400821 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200822 if (addr - block->offset < block->max_length) {
Paolo Bonzini041603f2013-09-09 17:49:45 +0200823 goto found;
824 }
825 }
826
827 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
828 abort();
829
830found:
Paolo Bonzini43771532013-09-09 17:58:40 +0200831 /* It is safe to write mru_block outside the iothread lock. This
832 * is what happens:
833 *
834 * mru_block = xxx
835 * rcu_read_unlock()
836 * xxx removed from list
837 * rcu_read_lock()
838 * read mru_block
839 * mru_block = NULL;
840 * call_rcu(reclaim_ramblock, xxx);
841 * rcu_read_unlock()
842 *
843 * atomic_rcu_set is not needed here. The block was already published
844 * when it was placed into the list. Here we're just making an extra
845 * copy of the pointer.
846 */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200847 ram_list.mru_block = block;
848 return block;
849}
850
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200851static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
bellard1ccde1c2004-02-06 19:46:14 +0000852{
Paolo Bonzini041603f2013-09-09 17:49:45 +0200853 ram_addr_t start1;
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200854 RAMBlock *block;
855 ram_addr_t end;
856
857 end = TARGET_PAGE_ALIGN(start + length);
858 start &= TARGET_PAGE_MASK;
bellardf23db162005-08-21 19:12:28 +0000859
Mike Day0dc3f442013-09-05 14:41:35 -0400860 rcu_read_lock();
Paolo Bonzini041603f2013-09-09 17:49:45 +0200861 block = qemu_get_ram_block(start);
862 assert(block == qemu_get_ram_block(end - 1));
Michael S. Tsirkin1240be22014-11-12 11:44:41 +0200863 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
Blue Swirle5548612012-04-21 13:08:33 +0000864 cpu_tlb_reset_dirty_all(start1, length);
Mike Day0dc3f442013-09-05 14:41:35 -0400865 rcu_read_unlock();
Juan Quintelad24981d2012-05-22 00:42:40 +0200866}
867
868/* Note: start and end must be within the same ram block. */
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000869bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
870 ram_addr_t length,
871 unsigned client)
Juan Quintelad24981d2012-05-22 00:42:40 +0200872{
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000873 unsigned long end, page;
874 bool dirty;
Juan Quintelad24981d2012-05-22 00:42:40 +0200875
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000876 if (length == 0) {
877 return false;
878 }
879
880 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
881 page = start >> TARGET_PAGE_BITS;
882 dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
883 page, end - page);
884
885 if (dirty && tcg_enabled()) {
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200886 tlb_reset_dirty_range_all(start, length);
Juan Quintelad24981d2012-05-22 00:42:40 +0200887 }
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000888
889 return dirty;
bellard1ccde1c2004-02-06 19:46:14 +0000890}
891
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100892/* Called from RCU critical section */
Andreas Färberbb0e6272013-09-03 13:32:01 +0200893hwaddr memory_region_section_get_iotlb(CPUState *cpu,
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200894 MemoryRegionSection *section,
895 target_ulong vaddr,
896 hwaddr paddr, hwaddr xlat,
897 int prot,
898 target_ulong *address)
Blue Swirle5548612012-04-21 13:08:33 +0000899{
Avi Kivitya8170e52012-10-23 12:30:10 +0200900 hwaddr iotlb;
Blue Swirle5548612012-04-21 13:08:33 +0000901 CPUWatchpoint *wp;
902
Blue Swirlcc5bea62012-04-14 14:56:48 +0000903 if (memory_region_is_ram(section->mr)) {
Blue Swirle5548612012-04-21 13:08:33 +0000904 /* Normal RAM. */
905 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200906 + xlat;
Blue Swirle5548612012-04-21 13:08:33 +0000907 if (!section->readonly) {
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200908 iotlb |= PHYS_SECTION_NOTDIRTY;
Blue Swirle5548612012-04-21 13:08:33 +0000909 } else {
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200910 iotlb |= PHYS_SECTION_ROM;
Blue Swirle5548612012-04-21 13:08:33 +0000911 }
912 } else {
Edgar E. Iglesias1b3fb982013-11-07 18:43:28 +0100913 iotlb = section - section->address_space->dispatch->map.sections;
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200914 iotlb += xlat;
Blue Swirle5548612012-04-21 13:08:33 +0000915 }
916
917 /* Make accesses to pages with watchpoints go via the
918 watchpoint trap routines. */
Andreas Färberff4700b2013-08-26 18:23:18 +0200919 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +0100920 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
Blue Swirle5548612012-04-21 13:08:33 +0000921 /* Avoid trapping reads of pages with a write breakpoint. */
922 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200923 iotlb = PHYS_SECTION_WATCH + paddr;
Blue Swirle5548612012-04-21 13:08:33 +0000924 *address |= TLB_MMIO;
925 break;
926 }
927 }
928 }
929
930 return iotlb;
931}
bellard9fa3e852004-01-04 18:06:42 +0000932#endif /* defined(CONFIG_USER_ONLY) */
933
pbrooke2eef172008-06-08 01:09:01 +0000934#if !defined(CONFIG_USER_ONLY)
pbrook8da3ff12008-12-01 18:59:50 +0000935
Anthony Liguoric227f092009-10-01 16:12:16 -0500936static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
Avi Kivity5312bd82012-02-12 18:32:55 +0200937 uint16_t section);
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200938static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
Avi Kivity54688b12012-02-09 17:34:32 +0200939
Igor Mammedova2b257d2014-10-31 16:38:37 +0000940static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
941 qemu_anon_ram_alloc;
Markus Armbruster91138032013-07-31 15:11:08 +0200942
943/*
944 * Set a custom physical guest memory alloator.
945 * Accelerators with unusual needs may need this. Hopefully, we can
946 * get rid of it eventually.
947 */
Igor Mammedova2b257d2014-10-31 16:38:37 +0000948void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
Markus Armbruster91138032013-07-31 15:11:08 +0200949{
950 phys_mem_alloc = alloc;
951}
952
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200953static uint16_t phys_section_add(PhysPageMap *map,
954 MemoryRegionSection *section)
Avi Kivity5312bd82012-02-12 18:32:55 +0200955{
Paolo Bonzini68f3f652013-05-07 11:30:23 +0200956 /* The physical section number is ORed with a page-aligned
957 * pointer to produce the iotlb entries. Thus it should
958 * never overflow into the page-aligned value.
959 */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200960 assert(map->sections_nb < TARGET_PAGE_SIZE);
Paolo Bonzini68f3f652013-05-07 11:30:23 +0200961
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200962 if (map->sections_nb == map->sections_nb_alloc) {
963 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
964 map->sections = g_renew(MemoryRegionSection, map->sections,
965 map->sections_nb_alloc);
Avi Kivity5312bd82012-02-12 18:32:55 +0200966 }
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200967 map->sections[map->sections_nb] = *section;
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200968 memory_region_ref(section->mr);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200969 return map->sections_nb++;
Avi Kivity5312bd82012-02-12 18:32:55 +0200970}
971
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200972static void phys_section_destroy(MemoryRegion *mr)
973{
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200974 memory_region_unref(mr);
975
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200976 if (mr->subpage) {
977 subpage_t *subpage = container_of(mr, subpage_t, iomem);
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -0700978 object_unref(OBJECT(&subpage->iomem));
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200979 g_free(subpage);
980 }
981}
982
Paolo Bonzini60926662013-05-29 12:30:26 +0200983static void phys_sections_free(PhysPageMap *map)
Avi Kivity5312bd82012-02-12 18:32:55 +0200984{
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200985 while (map->sections_nb > 0) {
986 MemoryRegionSection *section = &map->sections[--map->sections_nb];
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200987 phys_section_destroy(section->mr);
988 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200989 g_free(map->sections);
990 g_free(map->nodes);
Avi Kivity5312bd82012-02-12 18:32:55 +0200991}
992
Avi Kivityac1970f2012-10-03 16:22:53 +0200993static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +0200994{
995 subpage_t *subpage;
Avi Kivitya8170e52012-10-23 12:30:10 +0200996 hwaddr base = section->offset_within_address_space
Avi Kivity0f0cb162012-02-13 17:14:32 +0200997 & TARGET_PAGE_MASK;
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200998 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200999 d->map.nodes, d->map.sections);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001000 MemoryRegionSection subsection = {
1001 .offset_within_address_space = base,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001002 .size = int128_make64(TARGET_PAGE_SIZE),
Avi Kivity0f0cb162012-02-13 17:14:32 +02001003 };
Avi Kivitya8170e52012-10-23 12:30:10 +02001004 hwaddr start, end;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001005
Avi Kivityf3705d52012-03-08 16:16:34 +02001006 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001007
Avi Kivityf3705d52012-03-08 16:16:34 +02001008 if (!(existing->mr->subpage)) {
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001009 subpage = subpage_init(d->as, base);
Edgar E. Iglesias3be91e82013-11-07 18:42:51 +01001010 subsection.address_space = d->as;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001011 subsection.mr = &subpage->iomem;
Avi Kivityac1970f2012-10-03 16:22:53 +02001012 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001013 phys_section_add(&d->map, &subsection));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001014 } else {
Avi Kivityf3705d52012-03-08 16:16:34 +02001015 subpage = container_of(existing->mr, subpage_t, iomem);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001016 }
1017 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001018 end = start + int128_get64(section->size) - 1;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001019 subpage_register(subpage, start, end,
1020 phys_section_add(&d->map, section));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001021}
1022
1023
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001024static void register_multipage(AddressSpaceDispatch *d,
1025 MemoryRegionSection *section)
bellard33417e72003-08-10 21:47:01 +00001026{
Avi Kivitya8170e52012-10-23 12:30:10 +02001027 hwaddr start_addr = section->offset_within_address_space;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001028 uint16_t section_index = phys_section_add(&d->map, section);
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001029 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1030 TARGET_PAGE_BITS));
Avi Kivitydd811242012-01-02 12:17:03 +02001031
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001032 assert(num_pages);
1033 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
bellard33417e72003-08-10 21:47:01 +00001034}
1035
Avi Kivityac1970f2012-10-03 16:22:53 +02001036static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001037{
Paolo Bonzini89ae3372013-06-02 10:39:07 +02001038 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Paolo Bonzini00752702013-05-29 12:13:54 +02001039 AddressSpaceDispatch *d = as->next_dispatch;
Paolo Bonzini99b9cc02013-05-27 13:18:01 +02001040 MemoryRegionSection now = *section, remain = *section;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001041 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001042
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001043 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1044 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1045 - now.offset_within_address_space;
1046
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001047 now.size = int128_min(int128_make64(left), now.size);
Avi Kivityac1970f2012-10-03 16:22:53 +02001048 register_subpage(d, &now);
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001049 } else {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001050 now.size = int128_zero();
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001051 }
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001052 while (int128_ne(remain.size, now.size)) {
1053 remain.size = int128_sub(remain.size, now.size);
1054 remain.offset_within_address_space += int128_get64(now.size);
1055 remain.offset_within_region += int128_get64(now.size);
Tyler Hall69b67642012-07-25 18:45:04 -04001056 now = remain;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001057 if (int128_lt(remain.size, page_size)) {
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001058 register_subpage(d, &now);
Hu Tao88266242013-08-29 18:21:16 +08001059 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001060 now.size = page_size;
Avi Kivityac1970f2012-10-03 16:22:53 +02001061 register_subpage(d, &now);
Tyler Hall69b67642012-07-25 18:45:04 -04001062 } else {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001063 now.size = int128_and(now.size, int128_neg(page_size));
Avi Kivityac1970f2012-10-03 16:22:53 +02001064 register_multipage(d, &now);
Tyler Hall69b67642012-07-25 18:45:04 -04001065 }
Avi Kivity0f0cb162012-02-13 17:14:32 +02001066 }
1067}
1068
Sheng Yang62a27442010-01-26 19:21:16 +08001069void qemu_flush_coalesced_mmio_buffer(void)
1070{
1071 if (kvm_enabled())
1072 kvm_flush_coalesced_mmio_buffer();
1073}
1074
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001075void qemu_mutex_lock_ramlist(void)
1076{
1077 qemu_mutex_lock(&ram_list.mutex);
1078}
1079
1080void qemu_mutex_unlock_ramlist(void)
1081{
1082 qemu_mutex_unlock(&ram_list.mutex);
1083}
1084
Markus Armbrustere1e84ba2013-07-31 15:11:10 +02001085#ifdef __linux__
Marcelo Tosattic9027602010-03-01 20:25:08 -03001086
1087#include <sys/vfs.h>
1088
1089#define HUGETLBFS_MAGIC 0x958458f6
1090
Hu Taofc7a5802014-09-09 13:28:01 +08001091static long gethugepagesize(const char *path, Error **errp)
Marcelo Tosattic9027602010-03-01 20:25:08 -03001092{
1093 struct statfs fs;
1094 int ret;
1095
1096 do {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001097 ret = statfs(path, &fs);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001098 } while (ret != 0 && errno == EINTR);
1099
1100 if (ret != 0) {
Hu Taofc7a5802014-09-09 13:28:01 +08001101 error_setg_errno(errp, errno, "failed to get page size of file %s",
1102 path);
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001103 return 0;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001104 }
1105
1106 if (fs.f_type != HUGETLBFS_MAGIC)
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001107 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001108
1109 return fs.f_bsize;
1110}
1111
Alex Williamson04b16652010-07-02 11:13:17 -06001112static void *file_ram_alloc(RAMBlock *block,
1113 ram_addr_t memory,
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001114 const char *path,
1115 Error **errp)
Marcelo Tosattic9027602010-03-01 20:25:08 -03001116{
1117 char *filename;
Peter Feiner8ca761f2013-03-04 13:54:25 -05001118 char *sanitized_name;
1119 char *c;
Hu Tao557529d2014-09-09 13:28:00 +08001120 void *area = NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001121 int fd;
Hu Tao557529d2014-09-09 13:28:00 +08001122 uint64_t hpagesize;
Hu Taofc7a5802014-09-09 13:28:01 +08001123 Error *local_err = NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001124
Hu Taofc7a5802014-09-09 13:28:01 +08001125 hpagesize = gethugepagesize(path, &local_err);
1126 if (local_err) {
1127 error_propagate(errp, local_err);
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001128 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001129 }
Igor Mammedova2b257d2014-10-31 16:38:37 +00001130 block->mr->align = hpagesize;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001131
1132 if (memory < hpagesize) {
Hu Tao557529d2014-09-09 13:28:00 +08001133 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1134 "or larger than huge page size 0x%" PRIx64,
1135 memory, hpagesize);
1136 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001137 }
1138
1139 if (kvm_enabled() && !kvm_has_sync_mmu()) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001140 error_setg(errp,
1141 "host lacks kvm mmu notifiers, -mem-path unsupported");
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001142 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001143 }
1144
Peter Feiner8ca761f2013-03-04 13:54:25 -05001145 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
Peter Crosthwaite83234bf2014-08-14 23:54:29 -07001146 sanitized_name = g_strdup(memory_region_name(block->mr));
Peter Feiner8ca761f2013-03-04 13:54:25 -05001147 for (c = sanitized_name; *c != '\0'; c++) {
1148 if (*c == '/')
1149 *c = '_';
1150 }
1151
1152 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1153 sanitized_name);
1154 g_free(sanitized_name);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001155
1156 fd = mkstemp(filename);
1157 if (fd < 0) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001158 error_setg_errno(errp, errno,
1159 "unable to create backing store for hugepages");
Stefan Weile4ada482013-01-16 18:37:23 +01001160 g_free(filename);
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001161 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001162 }
1163 unlink(filename);
Stefan Weile4ada482013-01-16 18:37:23 +01001164 g_free(filename);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001165
1166 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1167
1168 /*
1169 * ftruncate is not supported by hugetlbfs in older
1170 * hosts, so don't bother bailing out on errors.
1171 * If anything goes wrong with it under other filesystems,
1172 * mmap will fail.
1173 */
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001174 if (ftruncate(fd, memory)) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001175 perror("ftruncate");
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001176 }
Marcelo Tosattic9027602010-03-01 20:25:08 -03001177
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001178 area = mmap(0, memory, PROT_READ | PROT_WRITE,
1179 (block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE),
1180 fd, 0);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001181 if (area == MAP_FAILED) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001182 error_setg_errno(errp, errno,
1183 "unable to map backing store for hugepages");
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001184 close(fd);
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001185 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001186 }
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001187
1188 if (mem_prealloc) {
Paolo Bonzini38183312014-05-14 17:43:21 +08001189 os_mem_prealloc(fd, area, memory);
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001190 }
1191
Alex Williamson04b16652010-07-02 11:13:17 -06001192 block->fd = fd;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001193 return area;
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001194
1195error:
1196 if (mem_prealloc) {
Gonglei81b07352015-02-25 12:22:31 +08001197 error_report("%s", error_get_pretty(*errp));
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001198 exit(1);
1199 }
1200 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001201}
1202#endif
1203
Mike Day0dc3f442013-09-05 14:41:35 -04001204/* Called with the ramlist lock held. */
Alex Williamsond17b5282010-06-25 11:08:38 -06001205static ram_addr_t find_ram_offset(ram_addr_t size)
1206{
Alex Williamson04b16652010-07-02 11:13:17 -06001207 RAMBlock *block, *next_block;
Alex Williamson3e837b22011-10-31 08:54:09 -06001208 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001209
Stefan Hajnoczi49cd9ac2013-03-11 10:20:21 +01001210 assert(size != 0); /* it would hand out same offset multiple times */
1211
Mike Day0dc3f442013-09-05 14:41:35 -04001212 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
Alex Williamson04b16652010-07-02 11:13:17 -06001213 return 0;
Mike Day0d53d9f2015-01-21 13:45:24 +01001214 }
Alex Williamson04b16652010-07-02 11:13:17 -06001215
Mike Day0dc3f442013-09-05 14:41:35 -04001216 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00001217 ram_addr_t end, next = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001218
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001219 end = block->offset + block->max_length;
Alex Williamson04b16652010-07-02 11:13:17 -06001220
Mike Day0dc3f442013-09-05 14:41:35 -04001221 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
Alex Williamson04b16652010-07-02 11:13:17 -06001222 if (next_block->offset >= end) {
1223 next = MIN(next, next_block->offset);
1224 }
1225 }
1226 if (next - end >= size && next - end < mingap) {
Alex Williamson3e837b22011-10-31 08:54:09 -06001227 offset = end;
Alex Williamson04b16652010-07-02 11:13:17 -06001228 mingap = next - end;
1229 }
1230 }
Alex Williamson3e837b22011-10-31 08:54:09 -06001231
1232 if (offset == RAM_ADDR_MAX) {
1233 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1234 (uint64_t)size);
1235 abort();
1236 }
1237
Alex Williamson04b16652010-07-02 11:13:17 -06001238 return offset;
1239}
1240
Juan Quintela652d7ec2012-07-20 10:37:54 +02001241ram_addr_t last_ram_offset(void)
Alex Williamson04b16652010-07-02 11:13:17 -06001242{
Alex Williamsond17b5282010-06-25 11:08:38 -06001243 RAMBlock *block;
1244 ram_addr_t last = 0;
1245
Mike Day0dc3f442013-09-05 14:41:35 -04001246 rcu_read_lock();
1247 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001248 last = MAX(last, block->offset + block->max_length);
Mike Day0d53d9f2015-01-21 13:45:24 +01001249 }
Mike Day0dc3f442013-09-05 14:41:35 -04001250 rcu_read_unlock();
Alex Williamsond17b5282010-06-25 11:08:38 -06001251 return last;
1252}
1253
Jason Baronddb97f12012-08-02 15:44:16 -04001254static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1255{
1256 int ret;
Jason Baronddb97f12012-08-02 15:44:16 -04001257
1258 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +02001259 if (!machine_dump_guest_core(current_machine)) {
Jason Baronddb97f12012-08-02 15:44:16 -04001260 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1261 if (ret) {
1262 perror("qemu_madvise");
1263 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1264 "but dump_guest_core=off specified\n");
1265 }
1266 }
1267}
1268
Mike Day0dc3f442013-09-05 14:41:35 -04001269/* Called within an RCU critical section, or while the ramlist lock
1270 * is held.
1271 */
Hu Tao20cfe882014-04-02 15:13:26 +08001272static RAMBlock *find_ram_block(ram_addr_t addr)
Cam Macdonell84b89d72010-07-26 18:10:57 -06001273{
Hu Tao20cfe882014-04-02 15:13:26 +08001274 RAMBlock *block;
Cam Macdonell84b89d72010-07-26 18:10:57 -06001275
Mike Day0dc3f442013-09-05 14:41:35 -04001276 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Avi Kivityc5705a72011-12-20 15:59:12 +02001277 if (block->offset == addr) {
Hu Tao20cfe882014-04-02 15:13:26 +08001278 return block;
Avi Kivityc5705a72011-12-20 15:59:12 +02001279 }
1280 }
Hu Tao20cfe882014-04-02 15:13:26 +08001281
1282 return NULL;
1283}
1284
Mike Dayae3a7042013-09-05 14:41:35 -04001285/* Called with iothread lock held. */
Hu Tao20cfe882014-04-02 15:13:26 +08001286void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1287{
Mike Dayae3a7042013-09-05 14:41:35 -04001288 RAMBlock *new_block, *block;
Hu Tao20cfe882014-04-02 15:13:26 +08001289
Mike Day0dc3f442013-09-05 14:41:35 -04001290 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001291 new_block = find_ram_block(addr);
Avi Kivityc5705a72011-12-20 15:59:12 +02001292 assert(new_block);
1293 assert(!new_block->idstr[0]);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001294
Anthony Liguori09e5ab62012-02-03 12:28:43 -06001295 if (dev) {
1296 char *id = qdev_get_dev_path(dev);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001297 if (id) {
1298 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
Anthony Liguori7267c092011-08-20 22:09:37 -05001299 g_free(id);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001300 }
1301 }
1302 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1303
Mike Day0dc3f442013-09-05 14:41:35 -04001304 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Avi Kivityc5705a72011-12-20 15:59:12 +02001305 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
Cam Macdonell84b89d72010-07-26 18:10:57 -06001306 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1307 new_block->idstr);
1308 abort();
1309 }
1310 }
Mike Day0dc3f442013-09-05 14:41:35 -04001311 rcu_read_unlock();
Avi Kivityc5705a72011-12-20 15:59:12 +02001312}
1313
Mike Dayae3a7042013-09-05 14:41:35 -04001314/* Called with iothread lock held. */
Hu Tao20cfe882014-04-02 15:13:26 +08001315void qemu_ram_unset_idstr(ram_addr_t addr)
1316{
Mike Dayae3a7042013-09-05 14:41:35 -04001317 RAMBlock *block;
Hu Tao20cfe882014-04-02 15:13:26 +08001318
Mike Dayae3a7042013-09-05 14:41:35 -04001319 /* FIXME: arch_init.c assumes that this is not called throughout
1320 * migration. Ignore the problem since hot-unplug during migration
1321 * does not work anyway.
1322 */
1323
Mike Day0dc3f442013-09-05 14:41:35 -04001324 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001325 block = find_ram_block(addr);
Hu Tao20cfe882014-04-02 15:13:26 +08001326 if (block) {
1327 memset(block->idstr, 0, sizeof(block->idstr));
1328 }
Mike Day0dc3f442013-09-05 14:41:35 -04001329 rcu_read_unlock();
Hu Tao20cfe882014-04-02 15:13:26 +08001330}
1331
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001332static int memory_try_enable_merging(void *addr, size_t len)
1333{
Marcel Apfelbaum75cc7f02015-02-04 17:43:55 +02001334 if (!machine_mem_merge(current_machine)) {
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001335 /* disabled by the user */
1336 return 0;
1337 }
1338
1339 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1340}
1341
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001342/* Only legal before guest might have detected the memory size: e.g. on
1343 * incoming migration, or right after reset.
1344 *
1345 * As memory core doesn't know how is memory accessed, it is up to
1346 * resize callback to update device state and/or add assertions to detect
1347 * misuse, if necessary.
1348 */
1349int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1350{
1351 RAMBlock *block = find_ram_block(base);
1352
1353 assert(block);
1354
Michael S. Tsirkin129ddaf2015-02-17 10:15:30 +01001355 newsize = TARGET_PAGE_ALIGN(newsize);
1356
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001357 if (block->used_length == newsize) {
1358 return 0;
1359 }
1360
1361 if (!(block->flags & RAM_RESIZEABLE)) {
1362 error_setg_errno(errp, EINVAL,
1363 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1364 " in != 0x" RAM_ADDR_FMT, block->idstr,
1365 newsize, block->used_length);
1366 return -EINVAL;
1367 }
1368
1369 if (block->max_length < newsize) {
1370 error_setg_errno(errp, EINVAL,
1371 "Length too large: %s: 0x" RAM_ADDR_FMT
1372 " > 0x" RAM_ADDR_FMT, block->idstr,
1373 newsize, block->max_length);
1374 return -EINVAL;
1375 }
1376
1377 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1378 block->used_length = newsize;
Paolo Bonzini58d27072015-03-23 11:56:01 +01001379 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1380 DIRTY_CLIENTS_ALL);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001381 memory_region_set_size(block->mr, newsize);
1382 if (block->resized) {
1383 block->resized(block->idstr, newsize, block->host);
1384 }
1385 return 0;
1386}
1387
Hu Taoef701d72014-09-09 13:27:54 +08001388static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
Avi Kivityc5705a72011-12-20 15:59:12 +02001389{
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001390 RAMBlock *block;
Mike Day0d53d9f2015-01-21 13:45:24 +01001391 RAMBlock *last_block = NULL;
Juan Quintela2152f5c2013-10-08 13:52:02 +02001392 ram_addr_t old_ram_size, new_ram_size;
1393
1394 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
Avi Kivityc5705a72011-12-20 15:59:12 +02001395
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001396 qemu_mutex_lock_ramlist();
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001397 new_block->offset = find_ram_offset(new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001398
1399 if (!new_block->host) {
1400 if (xen_enabled()) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001401 xen_ram_alloc(new_block->offset, new_block->max_length,
1402 new_block->mr);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001403 } else {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001404 new_block->host = phys_mem_alloc(new_block->max_length,
Igor Mammedova2b257d2014-10-31 16:38:37 +00001405 &new_block->mr->align);
Markus Armbruster39228252013-07-31 15:11:11 +02001406 if (!new_block->host) {
Hu Taoef701d72014-09-09 13:27:54 +08001407 error_setg_errno(errp, errno,
1408 "cannot set up guest memory '%s'",
1409 memory_region_name(new_block->mr));
1410 qemu_mutex_unlock_ramlist();
1411 return -1;
Markus Armbruster39228252013-07-31 15:11:11 +02001412 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001413 memory_try_enable_merging(new_block->host, new_block->max_length);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09001414 }
1415 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001416
Mike Day0d53d9f2015-01-21 13:45:24 +01001417 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1418 * QLIST (which has an RCU-friendly variant) does not have insertion at
1419 * tail, so save the last element in last_block.
1420 */
Mike Day0dc3f442013-09-05 14:41:35 -04001421 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Mike Day0d53d9f2015-01-21 13:45:24 +01001422 last_block = block;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001423 if (block->max_length < new_block->max_length) {
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001424 break;
1425 }
1426 }
1427 if (block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001428 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001429 } else if (last_block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001430 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001431 } else { /* list is empty */
Mike Day0dc3f442013-09-05 14:41:35 -04001432 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001433 }
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001434 ram_list.mru_block = NULL;
Cam Macdonell84b89d72010-07-26 18:10:57 -06001435
Mike Day0dc3f442013-09-05 14:41:35 -04001436 /* Write list before version */
1437 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001438 ram_list.version++;
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001439 qemu_mutex_unlock_ramlist();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001440
Juan Quintela2152f5c2013-10-08 13:52:02 +02001441 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1442
1443 if (new_ram_size > old_ram_size) {
Juan Quintela1ab4c8c2013-10-08 16:14:39 +02001444 int i;
Mike Dayae3a7042013-09-05 14:41:35 -04001445
1446 /* ram_list.dirty_memory[] is protected by the iothread lock. */
Juan Quintela1ab4c8c2013-10-08 16:14:39 +02001447 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1448 ram_list.dirty_memory[i] =
1449 bitmap_zero_extend(ram_list.dirty_memory[i],
1450 old_ram_size, new_ram_size);
1451 }
Juan Quintela2152f5c2013-10-08 13:52:02 +02001452 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001453 cpu_physical_memory_set_dirty_range(new_block->offset,
Paolo Bonzini58d27072015-03-23 11:56:01 +01001454 new_block->used_length,
1455 DIRTY_CLIENTS_ALL);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001456
Paolo Bonzinia904c912015-01-21 16:18:35 +01001457 if (new_block->host) {
1458 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1459 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1460 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1461 if (kvm_enabled()) {
1462 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1463 }
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001464 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001465
1466 return new_block->offset;
1467}
1468
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08001469#ifdef __linux__
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001470ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001471 bool share, const char *mem_path,
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001472 Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001473{
1474 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001475 ram_addr_t addr;
1476 Error *local_err = NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001477
1478 if (xen_enabled()) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001479 error_setg(errp, "-mem-path not supported with Xen");
1480 return -1;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001481 }
1482
1483 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1484 /*
1485 * file_ram_alloc() needs to allocate just like
1486 * phys_mem_alloc, but we haven't bothered to provide
1487 * a hook there.
1488 */
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001489 error_setg(errp,
1490 "-mem-path not supported with this accelerator");
1491 return -1;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001492 }
1493
1494 size = TARGET_PAGE_ALIGN(size);
1495 new_block = g_malloc0(sizeof(*new_block));
1496 new_block->mr = mr;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001497 new_block->used_length = size;
1498 new_block->max_length = size;
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001499 new_block->flags = share ? RAM_SHARED : 0;
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001500 new_block->host = file_ram_alloc(new_block, size,
1501 mem_path, errp);
1502 if (!new_block->host) {
1503 g_free(new_block);
1504 return -1;
1505 }
1506
Hu Taoef701d72014-09-09 13:27:54 +08001507 addr = ram_block_add(new_block, &local_err);
1508 if (local_err) {
1509 g_free(new_block);
1510 error_propagate(errp, local_err);
1511 return -1;
1512 }
1513 return addr;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001514}
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08001515#endif
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001516
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001517static
1518ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1519 void (*resized)(const char*,
1520 uint64_t length,
1521 void *host),
1522 void *host, bool resizeable,
Hu Taoef701d72014-09-09 13:27:54 +08001523 MemoryRegion *mr, Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001524{
1525 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001526 ram_addr_t addr;
1527 Error *local_err = NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001528
1529 size = TARGET_PAGE_ALIGN(size);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001530 max_size = TARGET_PAGE_ALIGN(max_size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001531 new_block = g_malloc0(sizeof(*new_block));
1532 new_block->mr = mr;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001533 new_block->resized = resized;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001534 new_block->used_length = size;
1535 new_block->max_length = max_size;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001536 assert(max_size >= size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001537 new_block->fd = -1;
1538 new_block->host = host;
1539 if (host) {
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08001540 new_block->flags |= RAM_PREALLOC;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001541 }
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001542 if (resizeable) {
1543 new_block->flags |= RAM_RESIZEABLE;
1544 }
Hu Taoef701d72014-09-09 13:27:54 +08001545 addr = ram_block_add(new_block, &local_err);
1546 if (local_err) {
1547 g_free(new_block);
1548 error_propagate(errp, local_err);
1549 return -1;
1550 }
1551 return addr;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001552}
1553
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001554ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1555 MemoryRegion *mr, Error **errp)
1556{
1557 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1558}
1559
Hu Taoef701d72014-09-09 13:27:54 +08001560ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
pbrook94a6b542009-04-11 17:15:54 +00001561{
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001562 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1563}
1564
1565ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1566 void (*resized)(const char*,
1567 uint64_t length,
1568 void *host),
1569 MemoryRegion *mr, Error **errp)
1570{
1571 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
pbrook94a6b542009-04-11 17:15:54 +00001572}
bellarde9a1ab12007-02-08 23:08:38 +00001573
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001574void qemu_ram_free_from_ptr(ram_addr_t addr)
1575{
1576 RAMBlock *block;
1577
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001578 qemu_mutex_lock_ramlist();
Mike Day0dc3f442013-09-05 14:41:35 -04001579 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001580 if (addr == block->offset) {
Mike Day0dc3f442013-09-05 14:41:35 -04001581 QLIST_REMOVE_RCU(block, next);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001582 ram_list.mru_block = NULL;
Mike Day0dc3f442013-09-05 14:41:35 -04001583 /* Write list before version */
1584 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001585 ram_list.version++;
Paolo Bonzini43771532013-09-09 17:58:40 +02001586 g_free_rcu(block, rcu);
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001587 break;
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001588 }
1589 }
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001590 qemu_mutex_unlock_ramlist();
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001591}
1592
Paolo Bonzini43771532013-09-09 17:58:40 +02001593static void reclaim_ramblock(RAMBlock *block)
1594{
1595 if (block->flags & RAM_PREALLOC) {
1596 ;
1597 } else if (xen_enabled()) {
1598 xen_invalidate_map_cache_entry(block->host);
1599#ifndef _WIN32
1600 } else if (block->fd >= 0) {
1601 munmap(block->host, block->max_length);
1602 close(block->fd);
1603#endif
1604 } else {
1605 qemu_anon_ram_free(block->host, block->max_length);
1606 }
1607 g_free(block);
1608}
1609
Anthony Liguoric227f092009-10-01 16:12:16 -05001610void qemu_ram_free(ram_addr_t addr)
bellarde9a1ab12007-02-08 23:08:38 +00001611{
Alex Williamson04b16652010-07-02 11:13:17 -06001612 RAMBlock *block;
1613
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001614 qemu_mutex_lock_ramlist();
Mike Day0dc3f442013-09-05 14:41:35 -04001615 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Alex Williamson04b16652010-07-02 11:13:17 -06001616 if (addr == block->offset) {
Mike Day0dc3f442013-09-05 14:41:35 -04001617 QLIST_REMOVE_RCU(block, next);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001618 ram_list.mru_block = NULL;
Mike Day0dc3f442013-09-05 14:41:35 -04001619 /* Write list before version */
1620 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001621 ram_list.version++;
Paolo Bonzini43771532013-09-09 17:58:40 +02001622 call_rcu(block, reclaim_ramblock, rcu);
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001623 break;
Alex Williamson04b16652010-07-02 11:13:17 -06001624 }
1625 }
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001626 qemu_mutex_unlock_ramlist();
bellarde9a1ab12007-02-08 23:08:38 +00001627}
1628
Huang Yingcd19cfa2011-03-02 08:56:19 +01001629#ifndef _WIN32
1630void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1631{
1632 RAMBlock *block;
1633 ram_addr_t offset;
1634 int flags;
1635 void *area, *vaddr;
1636
Mike Day0dc3f442013-09-05 14:41:35 -04001637 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01001638 offset = addr - block->offset;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001639 if (offset < block->max_length) {
Michael S. Tsirkin1240be22014-11-12 11:44:41 +02001640 vaddr = ramblock_ptr(block, offset);
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08001641 if (block->flags & RAM_PREALLOC) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01001642 ;
Markus Armbrusterdfeaf2a2013-07-31 15:11:05 +02001643 } else if (xen_enabled()) {
1644 abort();
Huang Yingcd19cfa2011-03-02 08:56:19 +01001645 } else {
1646 flags = MAP_FIXED;
Markus Armbruster3435f392013-07-31 15:11:07 +02001647 if (block->fd >= 0) {
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001648 flags |= (block->flags & RAM_SHARED ?
1649 MAP_SHARED : MAP_PRIVATE);
Markus Armbruster3435f392013-07-31 15:11:07 +02001650 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1651 flags, block->fd, offset);
Huang Yingcd19cfa2011-03-02 08:56:19 +01001652 } else {
Markus Armbruster2eb9fba2013-07-31 15:11:09 +02001653 /*
1654 * Remap needs to match alloc. Accelerators that
1655 * set phys_mem_alloc never remap. If they did,
1656 * we'd need a remap hook here.
1657 */
1658 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1659
Huang Yingcd19cfa2011-03-02 08:56:19 +01001660 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1661 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1662 flags, -1, 0);
Huang Yingcd19cfa2011-03-02 08:56:19 +01001663 }
1664 if (area != vaddr) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00001665 fprintf(stderr, "Could not remap addr: "
1666 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
Huang Yingcd19cfa2011-03-02 08:56:19 +01001667 length, addr);
1668 exit(1);
1669 }
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001670 memory_try_enable_merging(vaddr, length);
Jason Baronddb97f12012-08-02 15:44:16 -04001671 qemu_ram_setup_dump(vaddr, length);
Huang Yingcd19cfa2011-03-02 08:56:19 +01001672 }
Huang Yingcd19cfa2011-03-02 08:56:19 +01001673 }
1674 }
1675}
1676#endif /* !_WIN32 */
1677
Paolo Bonzinia35ba7b2014-06-10 19:15:23 +08001678int qemu_get_ram_fd(ram_addr_t addr)
1679{
Mike Dayae3a7042013-09-05 14:41:35 -04001680 RAMBlock *block;
1681 int fd;
Paolo Bonzinia35ba7b2014-06-10 19:15:23 +08001682
Mike Day0dc3f442013-09-05 14:41:35 -04001683 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001684 block = qemu_get_ram_block(addr);
1685 fd = block->fd;
Mike Day0dc3f442013-09-05 14:41:35 -04001686 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001687 return fd;
Paolo Bonzinia35ba7b2014-06-10 19:15:23 +08001688}
1689
Damjan Marion3fd74b82014-06-26 23:01:32 +02001690void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1691{
Mike Dayae3a7042013-09-05 14:41:35 -04001692 RAMBlock *block;
1693 void *ptr;
Damjan Marion3fd74b82014-06-26 23:01:32 +02001694
Mike Day0dc3f442013-09-05 14:41:35 -04001695 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001696 block = qemu_get_ram_block(addr);
1697 ptr = ramblock_ptr(block, 0);
Mike Day0dc3f442013-09-05 14:41:35 -04001698 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001699 return ptr;
Damjan Marion3fd74b82014-06-26 23:01:32 +02001700}
1701
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001702/* Return a host pointer to ram allocated with qemu_ram_alloc.
Mike Dayae3a7042013-09-05 14:41:35 -04001703 * This should not be used for general purpose DMA. Use address_space_map
1704 * or address_space_rw instead. For local memory (e.g. video ram) that the
1705 * device owns, use memory_region_get_ram_ptr.
Mike Day0dc3f442013-09-05 14:41:35 -04001706 *
1707 * By the time this function returns, the returned pointer is not protected
1708 * by RCU anymore. If the caller is not within an RCU critical section and
1709 * does not hold the iothread lock, it must have other means of protecting the
1710 * pointer, such as a reference to the region that includes the incoming
1711 * ram_addr_t.
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001712 */
1713void *qemu_get_ram_ptr(ram_addr_t addr)
1714{
Mike Dayae3a7042013-09-05 14:41:35 -04001715 RAMBlock *block;
1716 void *ptr;
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001717
Mike Day0dc3f442013-09-05 14:41:35 -04001718 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001719 block = qemu_get_ram_block(addr);
1720
1721 if (xen_enabled() && block->host == NULL) {
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001722 /* We need to check if the requested address is in the RAM
1723 * because we don't want to map the entire memory in QEMU.
1724 * In that case just map until the end of the page.
1725 */
1726 if (block->offset == 0) {
Mike Dayae3a7042013-09-05 14:41:35 -04001727 ptr = xen_map_cache(addr, 0, 0);
Mike Day0dc3f442013-09-05 14:41:35 -04001728 goto unlock;
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001729 }
Mike Dayae3a7042013-09-05 14:41:35 -04001730
1731 block->host = xen_map_cache(block->offset, block->max_length, 1);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001732 }
Mike Dayae3a7042013-09-05 14:41:35 -04001733 ptr = ramblock_ptr(block, addr - block->offset);
1734
Mike Day0dc3f442013-09-05 14:41:35 -04001735unlock:
1736 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001737 return ptr;
pbrookdc828ca2009-04-09 22:21:07 +00001738}
1739
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001740/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
Mike Dayae3a7042013-09-05 14:41:35 -04001741 * but takes a size argument.
Mike Day0dc3f442013-09-05 14:41:35 -04001742 *
1743 * By the time this function returns, the returned pointer is not protected
1744 * by RCU anymore. If the caller is not within an RCU critical section and
1745 * does not hold the iothread lock, it must have other means of protecting the
1746 * pointer, such as a reference to the region that includes the incoming
1747 * ram_addr_t.
Mike Dayae3a7042013-09-05 14:41:35 -04001748 */
Peter Maydellcb85f7a2013-07-08 09:44:04 +01001749static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001750{
Mike Dayae3a7042013-09-05 14:41:35 -04001751 void *ptr;
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01001752 if (*size == 0) {
1753 return NULL;
1754 }
Jan Kiszka868bb332011-06-21 22:59:09 +02001755 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02001756 return xen_map_cache(addr, *size, 1);
Jan Kiszka868bb332011-06-21 22:59:09 +02001757 } else {
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001758 RAMBlock *block;
Mike Day0dc3f442013-09-05 14:41:35 -04001759 rcu_read_lock();
1760 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001761 if (addr - block->offset < block->max_length) {
1762 if (addr - block->offset + *size > block->max_length)
1763 *size = block->max_length - addr + block->offset;
Mike Dayae3a7042013-09-05 14:41:35 -04001764 ptr = ramblock_ptr(block, addr - block->offset);
Mike Day0dc3f442013-09-05 14:41:35 -04001765 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001766 return ptr;
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001767 }
1768 }
1769
1770 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1771 abort();
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001772 }
1773}
1774
Paolo Bonzini7443b432013-06-03 12:44:02 +02001775/* Some of the softmmu routines need to translate from a host pointer
Mike Dayae3a7042013-09-05 14:41:35 -04001776 * (typically a TLB entry) back to a ram offset.
1777 *
1778 * By the time this function returns, the returned pointer is not protected
1779 * by RCU anymore. If the caller is not within an RCU critical section and
1780 * does not hold the iothread lock, it must have other means of protecting the
1781 * pointer, such as a reference to the region that includes the incoming
1782 * ram_addr_t.
1783 */
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001784MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
pbrook5579c7f2009-04-11 14:47:08 +00001785{
pbrook94a6b542009-04-11 17:15:54 +00001786 RAMBlock *block;
1787 uint8_t *host = ptr;
Mike Dayae3a7042013-09-05 14:41:35 -04001788 MemoryRegion *mr;
pbrook94a6b542009-04-11 17:15:54 +00001789
Jan Kiszka868bb332011-06-21 22:59:09 +02001790 if (xen_enabled()) {
Mike Day0dc3f442013-09-05 14:41:35 -04001791 rcu_read_lock();
Jan Kiszkae41d7c62011-06-21 22:59:08 +02001792 *ram_addr = xen_ram_addr_from_mapcache(ptr);
Mike Dayae3a7042013-09-05 14:41:35 -04001793 mr = qemu_get_ram_block(*ram_addr)->mr;
Mike Day0dc3f442013-09-05 14:41:35 -04001794 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001795 return mr;
Stefano Stabellini712c2b42011-05-19 18:35:46 +01001796 }
1797
Mike Day0dc3f442013-09-05 14:41:35 -04001798 rcu_read_lock();
1799 block = atomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001800 if (block && block->host && host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02001801 goto found;
1802 }
1803
Mike Day0dc3f442013-09-05 14:41:35 -04001804 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Jun Nakajima432d2682010-08-31 16:41:25 +01001805 /* This case append when the block is not mapped. */
1806 if (block->host == NULL) {
1807 continue;
1808 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001809 if (host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02001810 goto found;
Alex Williamsonf471a172010-06-11 11:11:42 -06001811 }
pbrook94a6b542009-04-11 17:15:54 +00001812 }
Jun Nakajima432d2682010-08-31 16:41:25 +01001813
Mike Day0dc3f442013-09-05 14:41:35 -04001814 rcu_read_unlock();
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001815 return NULL;
Paolo Bonzini23887b72013-05-06 14:28:39 +02001816
1817found:
1818 *ram_addr = block->offset + (host - block->host);
Mike Dayae3a7042013-09-05 14:41:35 -04001819 mr = block->mr;
Mike Day0dc3f442013-09-05 14:41:35 -04001820 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001821 return mr;
Marcelo Tosattie8902612010-10-11 15:31:19 -03001822}
Alex Williamsonf471a172010-06-11 11:11:42 -06001823
Avi Kivitya8170e52012-10-23 12:30:10 +02001824static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001825 uint64_t val, unsigned size)
bellard1ccde1c2004-02-06 19:46:14 +00001826{
Juan Quintela52159192013-10-08 12:44:04 +02001827 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001828 tb_invalidate_phys_page_fast(ram_addr, size);
bellard3a7d9292005-08-21 09:26:42 +00001829 }
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001830 switch (size) {
1831 case 1:
1832 stb_p(qemu_get_ram_ptr(ram_addr), val);
1833 break;
1834 case 2:
1835 stw_p(qemu_get_ram_ptr(ram_addr), val);
1836 break;
1837 case 4:
1838 stl_p(qemu_get_ram_ptr(ram_addr), val);
1839 break;
1840 default:
1841 abort();
1842 }
Paolo Bonzini58d27072015-03-23 11:56:01 +01001843 /* Set both VGA and migration bits for simplicity and to remove
1844 * the notdirty callback faster.
1845 */
1846 cpu_physical_memory_set_dirty_range(ram_addr, size,
1847 DIRTY_CLIENTS_NOCODE);
bellardf23db162005-08-21 19:12:28 +00001848 /* we remove the notdirty callback only if the code has been
1849 flushed */
Juan Quintelaa2cd8c82013-10-10 11:20:22 +02001850 if (!cpu_physical_memory_is_clean(ram_addr)) {
Andreas Färber4917cf42013-05-27 05:17:50 +02001851 CPUArchState *env = current_cpu->env_ptr;
Andreas Färber93afead2013-08-26 03:41:01 +02001852 tlb_set_dirty(env, current_cpu->mem_io_vaddr);
Andreas Färber4917cf42013-05-27 05:17:50 +02001853 }
bellard1ccde1c2004-02-06 19:46:14 +00001854}
1855
Paolo Bonzinib018ddf2013-05-24 14:48:38 +02001856static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1857 unsigned size, bool is_write)
1858{
1859 return is_write;
1860}
1861
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001862static const MemoryRegionOps notdirty_mem_ops = {
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001863 .write = notdirty_mem_write,
Paolo Bonzinib018ddf2013-05-24 14:48:38 +02001864 .valid.accepts = notdirty_mem_accepts,
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001865 .endianness = DEVICE_NATIVE_ENDIAN,
bellard1ccde1c2004-02-06 19:46:14 +00001866};
1867
pbrook0f459d12008-06-09 00:20:13 +00001868/* Generate a debug exception if a watchpoint has been hit. */
Peter Maydell66b9b432015-04-26 16:49:24 +01001869static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
pbrook0f459d12008-06-09 00:20:13 +00001870{
Andreas Färber93afead2013-08-26 03:41:01 +02001871 CPUState *cpu = current_cpu;
1872 CPUArchState *env = cpu->env_ptr;
aliguori06d55cc2008-11-18 20:24:06 +00001873 target_ulong pc, cs_base;
pbrook0f459d12008-06-09 00:20:13 +00001874 target_ulong vaddr;
aliguoria1d1bb32008-11-18 20:07:32 +00001875 CPUWatchpoint *wp;
aliguori06d55cc2008-11-18 20:24:06 +00001876 int cpu_flags;
pbrook0f459d12008-06-09 00:20:13 +00001877
Andreas Färberff4700b2013-08-26 18:23:18 +02001878 if (cpu->watchpoint_hit) {
aliguori06d55cc2008-11-18 20:24:06 +00001879 /* We re-entered the check after replacing the TB. Now raise
1880 * the debug interrupt so that is will trigger after the
1881 * current instruction. */
Andreas Färber93afead2013-08-26 03:41:01 +02001882 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
aliguori06d55cc2008-11-18 20:24:06 +00001883 return;
1884 }
Andreas Färber93afead2013-08-26 03:41:01 +02001885 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
Andreas Färberff4700b2013-08-26 18:23:18 +02001886 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +01001887 if (cpu_watchpoint_address_matches(wp, vaddr, len)
1888 && (wp->flags & flags)) {
Peter Maydell08225672014-09-12 14:06:48 +01001889 if (flags == BP_MEM_READ) {
1890 wp->flags |= BP_WATCHPOINT_HIT_READ;
1891 } else {
1892 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
1893 }
1894 wp->hitaddr = vaddr;
Peter Maydell66b9b432015-04-26 16:49:24 +01001895 wp->hitattrs = attrs;
Andreas Färberff4700b2013-08-26 18:23:18 +02001896 if (!cpu->watchpoint_hit) {
1897 cpu->watchpoint_hit = wp;
Andreas Färber239c51a2013-09-01 17:12:23 +02001898 tb_check_watchpoint(cpu);
aliguori6e140f22008-11-18 20:37:55 +00001899 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
Andreas Färber27103422013-08-26 08:31:06 +02001900 cpu->exception_index = EXCP_DEBUG;
Andreas Färber5638d182013-08-27 17:52:12 +02001901 cpu_loop_exit(cpu);
aliguori6e140f22008-11-18 20:37:55 +00001902 } else {
1903 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
Andreas Färber648f0342013-09-01 17:43:17 +02001904 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001905 cpu_resume_from_signal(cpu, NULL);
aliguori6e140f22008-11-18 20:37:55 +00001906 }
aliguori06d55cc2008-11-18 20:24:06 +00001907 }
aliguori6e140f22008-11-18 20:37:55 +00001908 } else {
1909 wp->flags &= ~BP_WATCHPOINT_HIT;
pbrook0f459d12008-06-09 00:20:13 +00001910 }
1911 }
1912}
1913
pbrook6658ffb2007-03-16 23:58:11 +00001914/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1915 so these check for a hit then pass through to the normal out-of-line
1916 phys routines. */
Peter Maydell66b9b432015-04-26 16:49:24 +01001917static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
1918 unsigned size, MemTxAttrs attrs)
pbrook6658ffb2007-03-16 23:58:11 +00001919{
Peter Maydell66b9b432015-04-26 16:49:24 +01001920 MemTxResult res;
1921 uint64_t data;
pbrook6658ffb2007-03-16 23:58:11 +00001922
Peter Maydell66b9b432015-04-26 16:49:24 +01001923 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
Avi Kivity1ec9b902012-01-02 12:47:48 +02001924 switch (size) {
Max Filippov67364152012-01-29 00:01:40 +04001925 case 1:
Peter Maydell66b9b432015-04-26 16:49:24 +01001926 data = address_space_ldub(&address_space_memory, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04001927 break;
1928 case 2:
Peter Maydell66b9b432015-04-26 16:49:24 +01001929 data = address_space_lduw(&address_space_memory, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04001930 break;
1931 case 4:
Peter Maydell66b9b432015-04-26 16:49:24 +01001932 data = address_space_ldl(&address_space_memory, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04001933 break;
Avi Kivity1ec9b902012-01-02 12:47:48 +02001934 default: abort();
1935 }
Peter Maydell66b9b432015-04-26 16:49:24 +01001936 *pdata = data;
1937 return res;
1938}
1939
1940static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
1941 uint64_t val, unsigned size,
1942 MemTxAttrs attrs)
1943{
1944 MemTxResult res;
1945
1946 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
1947 switch (size) {
1948 case 1:
1949 address_space_stb(&address_space_memory, addr, val, attrs, &res);
1950 break;
1951 case 2:
1952 address_space_stw(&address_space_memory, addr, val, attrs, &res);
1953 break;
1954 case 4:
1955 address_space_stl(&address_space_memory, addr, val, attrs, &res);
1956 break;
1957 default: abort();
1958 }
1959 return res;
pbrook6658ffb2007-03-16 23:58:11 +00001960}
1961
Avi Kivity1ec9b902012-01-02 12:47:48 +02001962static const MemoryRegionOps watch_mem_ops = {
Peter Maydell66b9b432015-04-26 16:49:24 +01001963 .read_with_attrs = watch_mem_read,
1964 .write_with_attrs = watch_mem_write,
Avi Kivity1ec9b902012-01-02 12:47:48 +02001965 .endianness = DEVICE_NATIVE_ENDIAN,
pbrook6658ffb2007-03-16 23:58:11 +00001966};
pbrook6658ffb2007-03-16 23:58:11 +00001967
Peter Maydellf25a49e2015-04-26 16:49:24 +01001968static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
1969 unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00001970{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001971 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01001972 uint8_t buf[8];
Peter Maydell5c9eb022015-04-26 16:49:24 +01001973 MemTxResult res;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02001974
blueswir1db7b5422007-05-26 17:36:03 +00001975#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08001976 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001977 subpage, len, addr);
blueswir1db7b5422007-05-26 17:36:03 +00001978#endif
Peter Maydell5c9eb022015-04-26 16:49:24 +01001979 res = address_space_read(subpage->as, addr + subpage->base,
1980 attrs, buf, len);
1981 if (res) {
1982 return res;
Peter Maydellf25a49e2015-04-26 16:49:24 +01001983 }
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001984 switch (len) {
1985 case 1:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001986 *data = ldub_p(buf);
1987 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001988 case 2:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001989 *data = lduw_p(buf);
1990 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001991 case 4:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001992 *data = ldl_p(buf);
1993 return MEMTX_OK;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01001994 case 8:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001995 *data = ldq_p(buf);
1996 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001997 default:
1998 abort();
1999 }
blueswir1db7b5422007-05-26 17:36:03 +00002000}
2001
Peter Maydellf25a49e2015-04-26 16:49:24 +01002002static MemTxResult subpage_write(void *opaque, hwaddr addr,
2003 uint64_t value, unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00002004{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002005 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002006 uint8_t buf[8];
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002007
blueswir1db7b5422007-05-26 17:36:03 +00002008#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002009 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002010 " value %"PRIx64"\n",
2011 __func__, subpage, len, addr, value);
blueswir1db7b5422007-05-26 17:36:03 +00002012#endif
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002013 switch (len) {
2014 case 1:
2015 stb_p(buf, value);
2016 break;
2017 case 2:
2018 stw_p(buf, value);
2019 break;
2020 case 4:
2021 stl_p(buf, value);
2022 break;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002023 case 8:
2024 stq_p(buf, value);
2025 break;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002026 default:
2027 abort();
2028 }
Peter Maydell5c9eb022015-04-26 16:49:24 +01002029 return address_space_write(subpage->as, addr + subpage->base,
2030 attrs, buf, len);
blueswir1db7b5422007-05-26 17:36:03 +00002031}
2032
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002033static bool subpage_accepts(void *opaque, hwaddr addr,
Amos Kong016e9d62013-09-27 09:25:38 +08002034 unsigned len, bool is_write)
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002035{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002036 subpage_t *subpage = opaque;
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002037#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002038 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002039 __func__, subpage, is_write ? 'w' : 'r', len, addr);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002040#endif
2041
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002042 return address_space_access_valid(subpage->as, addr + subpage->base,
Amos Kong016e9d62013-09-27 09:25:38 +08002043 len, is_write);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002044}
2045
Avi Kivity70c68e42012-01-02 12:32:48 +02002046static const MemoryRegionOps subpage_ops = {
Peter Maydellf25a49e2015-04-26 16:49:24 +01002047 .read_with_attrs = subpage_read,
2048 .write_with_attrs = subpage_write,
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002049 .impl.min_access_size = 1,
2050 .impl.max_access_size = 8,
2051 .valid.min_access_size = 1,
2052 .valid.max_access_size = 8,
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002053 .valid.accepts = subpage_accepts,
Avi Kivity70c68e42012-01-02 12:32:48 +02002054 .endianness = DEVICE_NATIVE_ENDIAN,
blueswir1db7b5422007-05-26 17:36:03 +00002055};
2056
Anthony Liguoric227f092009-10-01 16:12:16 -05002057static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
Avi Kivity5312bd82012-02-12 18:32:55 +02002058 uint16_t section)
blueswir1db7b5422007-05-26 17:36:03 +00002059{
2060 int idx, eidx;
2061
2062 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2063 return -1;
2064 idx = SUBPAGE_IDX(start);
2065 eidx = SUBPAGE_IDX(end);
2066#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002067 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2068 __func__, mmio, start, end, idx, eidx, section);
blueswir1db7b5422007-05-26 17:36:03 +00002069#endif
blueswir1db7b5422007-05-26 17:36:03 +00002070 for (; idx <= eidx; idx++) {
Avi Kivity5312bd82012-02-12 18:32:55 +02002071 mmio->sub_section[idx] = section;
blueswir1db7b5422007-05-26 17:36:03 +00002072 }
2073
2074 return 0;
2075}
2076
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002077static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
blueswir1db7b5422007-05-26 17:36:03 +00002078{
Anthony Liguoric227f092009-10-01 16:12:16 -05002079 subpage_t *mmio;
blueswir1db7b5422007-05-26 17:36:03 +00002080
Anthony Liguori7267c092011-08-20 22:09:37 -05002081 mmio = g_malloc0(sizeof(subpage_t));
aliguori1eec6142009-02-05 22:06:18 +00002082
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002083 mmio->as = as;
aliguori1eec6142009-02-05 22:06:18 +00002084 mmio->base = base;
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002085 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07002086 NULL, TARGET_PAGE_SIZE);
Avi Kivityb3b00c72012-01-02 13:20:11 +02002087 mmio->iomem.subpage = true;
blueswir1db7b5422007-05-26 17:36:03 +00002088#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002089 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2090 mmio, base, TARGET_PAGE_SIZE);
blueswir1db7b5422007-05-26 17:36:03 +00002091#endif
Liu Ping Fanb41aac42013-05-29 11:09:17 +02002092 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
blueswir1db7b5422007-05-26 17:36:03 +00002093
2094 return mmio;
2095}
2096
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002097static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2098 MemoryRegion *mr)
Avi Kivity5312bd82012-02-12 18:32:55 +02002099{
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002100 assert(as);
Avi Kivity5312bd82012-02-12 18:32:55 +02002101 MemoryRegionSection section = {
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002102 .address_space = as,
Avi Kivity5312bd82012-02-12 18:32:55 +02002103 .mr = mr,
2104 .offset_within_address_space = 0,
2105 .offset_within_region = 0,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02002106 .size = int128_2_64(),
Avi Kivity5312bd82012-02-12 18:32:55 +02002107 };
2108
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002109 return phys_section_add(map, &section);
Avi Kivity5312bd82012-02-12 18:32:55 +02002110}
2111
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +02002112MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
Avi Kivityaa102232012-03-08 17:06:55 +02002113{
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002114 AddressSpaceDispatch *d = atomic_rcu_read(&cpu->memory_dispatch);
2115 MemoryRegionSection *sections = d->map.sections;
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +02002116
2117 return sections[index & ~TARGET_PAGE_MASK].mr;
Avi Kivityaa102232012-03-08 17:06:55 +02002118}
2119
Avi Kivitye9179ce2009-06-14 11:38:52 +03002120static void io_mem_init(void)
2121{
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002122 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002123 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002124 NULL, UINT64_MAX);
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002125 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002126 NULL, UINT64_MAX);
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002127 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002128 NULL, UINT64_MAX);
Avi Kivitye9179ce2009-06-14 11:38:52 +03002129}
2130
Avi Kivityac1970f2012-10-03 16:22:53 +02002131static void mem_begin(MemoryListener *listener)
2132{
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002133 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002134 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2135 uint16_t n;
2136
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002137 n = dummy_section(&d->map, as, &io_mem_unassigned);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002138 assert(n == PHYS_SECTION_UNASSIGNED);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002139 n = dummy_section(&d->map, as, &io_mem_notdirty);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002140 assert(n == PHYS_SECTION_NOTDIRTY);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002141 n = dummy_section(&d->map, as, &io_mem_rom);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002142 assert(n == PHYS_SECTION_ROM);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002143 n = dummy_section(&d->map, as, &io_mem_watch);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002144 assert(n == PHYS_SECTION_WATCH);
Paolo Bonzini00752702013-05-29 12:13:54 +02002145
Michael S. Tsirkin9736e552013-11-11 14:42:43 +02002146 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
Paolo Bonzini00752702013-05-29 12:13:54 +02002147 d->as = as;
2148 as->next_dispatch = d;
2149}
2150
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002151static void address_space_dispatch_free(AddressSpaceDispatch *d)
2152{
2153 phys_sections_free(&d->map);
2154 g_free(d);
2155}
2156
Paolo Bonzini00752702013-05-29 12:13:54 +02002157static void mem_commit(MemoryListener *listener)
2158{
2159 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Paolo Bonzini0475d942013-05-29 12:28:21 +02002160 AddressSpaceDispatch *cur = as->dispatch;
2161 AddressSpaceDispatch *next = as->next_dispatch;
Avi Kivityac1970f2012-10-03 16:22:53 +02002162
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002163 phys_page_compact_all(next, next->map.nodes_nb);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +02002164
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002165 atomic_rcu_set(&as->dispatch, next);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002166 if (cur) {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002167 call_rcu(cur, address_space_dispatch_free, rcu);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002168 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02002169}
2170
Avi Kivity1d711482012-10-02 18:54:45 +02002171static void tcg_commit(MemoryListener *listener)
Avi Kivity50c1e142012-02-08 21:36:02 +02002172{
Andreas Färber182735e2013-05-29 22:29:20 +02002173 CPUState *cpu;
Avi Kivity117712c2012-02-12 21:23:17 +02002174
2175 /* since each CPU stores ram addresses in its TLB cache, we must
2176 reset the modified entries */
2177 /* XXX: slow ! */
Andreas Färberbdc44642013-06-24 23:50:24 +02002178 CPU_FOREACH(cpu) {
Edgar E. Iglesias33bde2e2013-11-21 19:06:30 +01002179 /* FIXME: Disentangle the cpu.h circular files deps so we can
2180 directly get the right CPU from listener. */
2181 if (cpu->tcg_as_listener != listener) {
2182 continue;
2183 }
Paolo Bonzini76e5c762015-01-15 12:46:47 +01002184 cpu_reload_memory_map(cpu);
Avi Kivity117712c2012-02-12 21:23:17 +02002185 }
Avi Kivity50c1e142012-02-08 21:36:02 +02002186}
2187
Avi Kivityac1970f2012-10-03 16:22:53 +02002188void address_space_init_dispatch(AddressSpace *as)
2189{
Paolo Bonzini00752702013-05-29 12:13:54 +02002190 as->dispatch = NULL;
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002191 as->dispatch_listener = (MemoryListener) {
Avi Kivityac1970f2012-10-03 16:22:53 +02002192 .begin = mem_begin,
Paolo Bonzini00752702013-05-29 12:13:54 +02002193 .commit = mem_commit,
Avi Kivityac1970f2012-10-03 16:22:53 +02002194 .region_add = mem_add,
2195 .region_nop = mem_add,
2196 .priority = 0,
2197 };
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002198 memory_listener_register(&as->dispatch_listener, as);
Avi Kivityac1970f2012-10-03 16:22:53 +02002199}
2200
Paolo Bonzini6e48e8f2015-02-10 10:25:44 -07002201void address_space_unregister(AddressSpace *as)
2202{
2203 memory_listener_unregister(&as->dispatch_listener);
2204}
2205
Avi Kivity83f3c252012-10-07 12:59:55 +02002206void address_space_destroy_dispatch(AddressSpace *as)
2207{
2208 AddressSpaceDispatch *d = as->dispatch;
2209
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002210 atomic_rcu_set(&as->dispatch, NULL);
2211 if (d) {
2212 call_rcu(d, address_space_dispatch_free, rcu);
2213 }
Avi Kivity83f3c252012-10-07 12:59:55 +02002214}
2215
Avi Kivity62152b82011-07-26 14:26:14 +03002216static void memory_map_init(void)
2217{
Anthony Liguori7267c092011-08-20 22:09:37 -05002218 system_memory = g_malloc(sizeof(*system_memory));
Paolo Bonzini03f49952013-11-07 17:14:36 +01002219
Paolo Bonzini57271d62013-11-07 17:14:37 +01002220 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002221 address_space_init(&address_space_memory, system_memory, "memory");
Avi Kivity309cb472011-08-08 16:09:03 +03002222
Anthony Liguori7267c092011-08-20 22:09:37 -05002223 system_io = g_malloc(sizeof(*system_io));
Jan Kiszka3bb28b72013-09-02 18:43:30 +02002224 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2225 65536);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002226 address_space_init(&address_space_io, system_io, "I/O");
Avi Kivity62152b82011-07-26 14:26:14 +03002227}
2228
2229MemoryRegion *get_system_memory(void)
2230{
2231 return system_memory;
2232}
2233
Avi Kivity309cb472011-08-08 16:09:03 +03002234MemoryRegion *get_system_io(void)
2235{
2236 return system_io;
2237}
2238
pbrooke2eef172008-06-08 01:09:01 +00002239#endif /* !defined(CONFIG_USER_ONLY) */
2240
bellard13eb76e2004-01-24 15:23:36 +00002241/* physical memory access (slow version, mainly for debug) */
2242#if defined(CONFIG_USER_ONLY)
Andreas Färberf17ec442013-06-29 19:40:58 +02002243int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
Paul Brooka68fe892010-03-01 00:08:59 +00002244 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00002245{
2246 int l, flags;
2247 target_ulong page;
pbrook53a59602006-03-25 19:31:22 +00002248 void * p;
bellard13eb76e2004-01-24 15:23:36 +00002249
2250 while (len > 0) {
2251 page = addr & TARGET_PAGE_MASK;
2252 l = (page + TARGET_PAGE_SIZE) - addr;
2253 if (l > len)
2254 l = len;
2255 flags = page_get_flags(page);
2256 if (!(flags & PAGE_VALID))
Paul Brooka68fe892010-03-01 00:08:59 +00002257 return -1;
bellard13eb76e2004-01-24 15:23:36 +00002258 if (is_write) {
2259 if (!(flags & PAGE_WRITE))
Paul Brooka68fe892010-03-01 00:08:59 +00002260 return -1;
bellard579a97f2007-11-11 14:26:47 +00002261 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00002262 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
Paul Brooka68fe892010-03-01 00:08:59 +00002263 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00002264 memcpy(p, buf, l);
2265 unlock_user(p, addr, l);
bellard13eb76e2004-01-24 15:23:36 +00002266 } else {
2267 if (!(flags & PAGE_READ))
Paul Brooka68fe892010-03-01 00:08:59 +00002268 return -1;
bellard579a97f2007-11-11 14:26:47 +00002269 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00002270 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
Paul Brooka68fe892010-03-01 00:08:59 +00002271 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00002272 memcpy(buf, p, l);
aurel325b257572008-04-28 08:54:59 +00002273 unlock_user(p, addr, 0);
bellard13eb76e2004-01-24 15:23:36 +00002274 }
2275 len -= l;
2276 buf += l;
2277 addr += l;
2278 }
Paul Brooka68fe892010-03-01 00:08:59 +00002279 return 0;
bellard13eb76e2004-01-24 15:23:36 +00002280}
bellard8df1cd02005-01-28 22:37:22 +00002281
bellard13eb76e2004-01-24 15:23:36 +00002282#else
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002283
Paolo Bonzini845b6212015-03-23 11:45:53 +01002284static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
Avi Kivitya8170e52012-10-23 12:30:10 +02002285 hwaddr length)
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002286{
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002287 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2288 /* No early return if dirty_log_mask is or becomes 0, because
2289 * cpu_physical_memory_set_dirty_range will still call
2290 * xen_modified_memory.
2291 */
2292 if (dirty_log_mask) {
2293 dirty_log_mask =
2294 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002295 }
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002296 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2297 tb_invalidate_phys_range(addr, addr + length);
2298 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2299 }
2300 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002301}
2302
Richard Henderson23326162013-07-08 14:55:59 -07002303static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
Paolo Bonzini82f25632013-05-24 11:59:43 +02002304{
Paolo Bonzinie1622f42013-07-17 13:17:41 +02002305 unsigned access_size_max = mr->ops->valid.max_access_size;
Richard Henderson23326162013-07-08 14:55:59 -07002306
2307 /* Regions are assumed to support 1-4 byte accesses unless
2308 otherwise specified. */
Richard Henderson23326162013-07-08 14:55:59 -07002309 if (access_size_max == 0) {
2310 access_size_max = 4;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002311 }
Richard Henderson23326162013-07-08 14:55:59 -07002312
2313 /* Bound the maximum access by the alignment of the address. */
2314 if (!mr->ops->impl.unaligned) {
2315 unsigned align_size_max = addr & -addr;
2316 if (align_size_max != 0 && align_size_max < access_size_max) {
2317 access_size_max = align_size_max;
2318 }
2319 }
2320
2321 /* Don't attempt accesses larger than the maximum. */
2322 if (l > access_size_max) {
2323 l = access_size_max;
2324 }
Paolo Bonzini098178f2013-07-29 14:27:39 +02002325 if (l & (l - 1)) {
2326 l = 1 << (qemu_fls(l) - 1);
2327 }
Richard Henderson23326162013-07-08 14:55:59 -07002328
2329 return l;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002330}
2331
Jan Kiszka4840f102015-06-18 18:47:22 +02002332static bool prepare_mmio_access(MemoryRegion *mr)
Paolo Bonzini125b3802015-06-18 18:47:21 +02002333{
Jan Kiszka4840f102015-06-18 18:47:22 +02002334 bool unlocked = !qemu_mutex_iothread_locked();
2335 bool release_lock = false;
2336
2337 if (unlocked && mr->global_locking) {
2338 qemu_mutex_lock_iothread();
2339 unlocked = false;
2340 release_lock = true;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002341 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002342 if (mr->flush_coalesced_mmio) {
2343 if (unlocked) {
2344 qemu_mutex_lock_iothread();
2345 }
2346 qemu_flush_coalesced_mmio_buffer();
2347 if (unlocked) {
2348 qemu_mutex_unlock_iothread();
2349 }
2350 }
2351
2352 return release_lock;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002353}
2354
Peter Maydell5c9eb022015-04-26 16:49:24 +01002355MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2356 uint8_t *buf, int len, bool is_write)
bellard13eb76e2004-01-24 15:23:36 +00002357{
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002358 hwaddr l;
bellard13eb76e2004-01-24 15:23:36 +00002359 uint8_t *ptr;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002360 uint64_t val;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002361 hwaddr addr1;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002362 MemoryRegion *mr;
Peter Maydell3b643492015-04-26 16:49:23 +01002363 MemTxResult result = MEMTX_OK;
Jan Kiszka4840f102015-06-18 18:47:22 +02002364 bool release_lock = false;
ths3b46e622007-09-17 08:09:54 +00002365
Paolo Bonzini41063e12015-03-18 14:21:43 +01002366 rcu_read_lock();
bellard13eb76e2004-01-24 15:23:36 +00002367 while (len > 0) {
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002368 l = len;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002369 mr = address_space_translate(as, addr, &addr1, &l, is_write);
ths3b46e622007-09-17 08:09:54 +00002370
bellard13eb76e2004-01-24 15:23:36 +00002371 if (is_write) {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002372 if (!memory_access_is_direct(mr, is_write)) {
Jan Kiszka4840f102015-06-18 18:47:22 +02002373 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002374 l = memory_access_size(mr, l, addr1);
Andreas Färber4917cf42013-05-27 05:17:50 +02002375 /* XXX: could force current_cpu to NULL to avoid
bellard6a00d602005-11-21 23:25:50 +00002376 potential bugs */
Richard Henderson23326162013-07-08 14:55:59 -07002377 switch (l) {
2378 case 8:
2379 /* 64 bit write access */
2380 val = ldq_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002381 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2382 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002383 break;
2384 case 4:
bellard1c213d12005-09-03 10:49:04 +00002385 /* 32 bit write access */
bellardc27004e2005-01-03 23:35:10 +00002386 val = ldl_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002387 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2388 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002389 break;
2390 case 2:
bellard1c213d12005-09-03 10:49:04 +00002391 /* 16 bit write access */
bellardc27004e2005-01-03 23:35:10 +00002392 val = lduw_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002393 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2394 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002395 break;
2396 case 1:
bellard1c213d12005-09-03 10:49:04 +00002397 /* 8 bit write access */
bellardc27004e2005-01-03 23:35:10 +00002398 val = ldub_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002399 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2400 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002401 break;
2402 default:
2403 abort();
bellard13eb76e2004-01-24 15:23:36 +00002404 }
Paolo Bonzini2bbfa052013-05-24 12:29:54 +02002405 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002406 addr1 += memory_region_get_ram_addr(mr);
bellard13eb76e2004-01-24 15:23:36 +00002407 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00002408 ptr = qemu_get_ram_ptr(addr1);
bellard13eb76e2004-01-24 15:23:36 +00002409 memcpy(ptr, buf, l);
Paolo Bonzini845b6212015-03-23 11:45:53 +01002410 invalidate_and_set_dirty(mr, addr1, l);
bellard13eb76e2004-01-24 15:23:36 +00002411 }
2412 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002413 if (!memory_access_is_direct(mr, is_write)) {
bellard13eb76e2004-01-24 15:23:36 +00002414 /* I/O case */
Jan Kiszka4840f102015-06-18 18:47:22 +02002415 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002416 l = memory_access_size(mr, l, addr1);
Richard Henderson23326162013-07-08 14:55:59 -07002417 switch (l) {
2418 case 8:
2419 /* 64 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002420 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2421 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002422 stq_p(buf, val);
2423 break;
2424 case 4:
bellard13eb76e2004-01-24 15:23:36 +00002425 /* 32 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002426 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2427 attrs);
bellardc27004e2005-01-03 23:35:10 +00002428 stl_p(buf, val);
Richard Henderson23326162013-07-08 14:55:59 -07002429 break;
2430 case 2:
bellard13eb76e2004-01-24 15:23:36 +00002431 /* 16 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002432 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2433 attrs);
bellardc27004e2005-01-03 23:35:10 +00002434 stw_p(buf, val);
Richard Henderson23326162013-07-08 14:55:59 -07002435 break;
2436 case 1:
bellard1c213d12005-09-03 10:49:04 +00002437 /* 8 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002438 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2439 attrs);
bellardc27004e2005-01-03 23:35:10 +00002440 stb_p(buf, val);
Richard Henderson23326162013-07-08 14:55:59 -07002441 break;
2442 default:
2443 abort();
bellard13eb76e2004-01-24 15:23:36 +00002444 }
2445 } else {
2446 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002447 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
Avi Kivityf3705d52012-03-08 16:16:34 +02002448 memcpy(buf, ptr, l);
bellard13eb76e2004-01-24 15:23:36 +00002449 }
2450 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002451
2452 if (release_lock) {
2453 qemu_mutex_unlock_iothread();
2454 release_lock = false;
2455 }
2456
bellard13eb76e2004-01-24 15:23:36 +00002457 len -= l;
2458 buf += l;
2459 addr += l;
2460 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002461 rcu_read_unlock();
Paolo Bonzinifd8aaa72013-05-21 09:56:55 +02002462
Peter Maydell3b643492015-04-26 16:49:23 +01002463 return result;
bellard13eb76e2004-01-24 15:23:36 +00002464}
bellard8df1cd02005-01-28 22:37:22 +00002465
Peter Maydell5c9eb022015-04-26 16:49:24 +01002466MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2467 const uint8_t *buf, int len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002468{
Peter Maydell5c9eb022015-04-26 16:49:24 +01002469 return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
Avi Kivityac1970f2012-10-03 16:22:53 +02002470}
2471
Peter Maydell5c9eb022015-04-26 16:49:24 +01002472MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2473 uint8_t *buf, int len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002474{
Peter Maydell5c9eb022015-04-26 16:49:24 +01002475 return address_space_rw(as, addr, attrs, buf, len, false);
Avi Kivityac1970f2012-10-03 16:22:53 +02002476}
2477
2478
Avi Kivitya8170e52012-10-23 12:30:10 +02002479void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
Avi Kivityac1970f2012-10-03 16:22:53 +02002480 int len, int is_write)
2481{
Peter Maydell5c9eb022015-04-26 16:49:24 +01002482 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2483 buf, len, is_write);
Avi Kivityac1970f2012-10-03 16:22:53 +02002484}
2485
Alexander Graf582b55a2013-12-11 14:17:44 +01002486enum write_rom_type {
2487 WRITE_DATA,
2488 FLUSH_CACHE,
2489};
2490
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002491static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
Alexander Graf582b55a2013-12-11 14:17:44 +01002492 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
bellardd0ecd2a2006-04-23 17:14:48 +00002493{
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002494 hwaddr l;
bellardd0ecd2a2006-04-23 17:14:48 +00002495 uint8_t *ptr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002496 hwaddr addr1;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002497 MemoryRegion *mr;
ths3b46e622007-09-17 08:09:54 +00002498
Paolo Bonzini41063e12015-03-18 14:21:43 +01002499 rcu_read_lock();
bellardd0ecd2a2006-04-23 17:14:48 +00002500 while (len > 0) {
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002501 l = len;
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002502 mr = address_space_translate(as, addr, &addr1, &l, true);
ths3b46e622007-09-17 08:09:54 +00002503
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002504 if (!(memory_region_is_ram(mr) ||
2505 memory_region_is_romd(mr))) {
Paolo Bonzinib242e0e2015-07-04 00:24:51 +02002506 l = memory_access_size(mr, l, addr1);
bellardd0ecd2a2006-04-23 17:14:48 +00002507 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002508 addr1 += memory_region_get_ram_addr(mr);
bellardd0ecd2a2006-04-23 17:14:48 +00002509 /* ROM/RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00002510 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf582b55a2013-12-11 14:17:44 +01002511 switch (type) {
2512 case WRITE_DATA:
2513 memcpy(ptr, buf, l);
Paolo Bonzini845b6212015-03-23 11:45:53 +01002514 invalidate_and_set_dirty(mr, addr1, l);
Alexander Graf582b55a2013-12-11 14:17:44 +01002515 break;
2516 case FLUSH_CACHE:
2517 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2518 break;
2519 }
bellardd0ecd2a2006-04-23 17:14:48 +00002520 }
2521 len -= l;
2522 buf += l;
2523 addr += l;
2524 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002525 rcu_read_unlock();
bellardd0ecd2a2006-04-23 17:14:48 +00002526}
2527
Alexander Graf582b55a2013-12-11 14:17:44 +01002528/* used for ROM loading : can write in RAM and ROM */
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002529void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
Alexander Graf582b55a2013-12-11 14:17:44 +01002530 const uint8_t *buf, int len)
2531{
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002532 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
Alexander Graf582b55a2013-12-11 14:17:44 +01002533}
2534
2535void cpu_flush_icache_range(hwaddr start, int len)
2536{
2537 /*
2538 * This function should do the same thing as an icache flush that was
2539 * triggered from within the guest. For TCG we are always cache coherent,
2540 * so there is no need to flush anything. For KVM / Xen we need to flush
2541 * the host's instruction cache at least.
2542 */
2543 if (tcg_enabled()) {
2544 return;
2545 }
2546
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002547 cpu_physical_memory_write_rom_internal(&address_space_memory,
2548 start, NULL, len, FLUSH_CACHE);
Alexander Graf582b55a2013-12-11 14:17:44 +01002549}
2550
aliguori6d16c2f2009-01-22 16:59:11 +00002551typedef struct {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002552 MemoryRegion *mr;
aliguori6d16c2f2009-01-22 16:59:11 +00002553 void *buffer;
Avi Kivitya8170e52012-10-23 12:30:10 +02002554 hwaddr addr;
2555 hwaddr len;
Fam Zhengc2cba0f2015-03-16 17:03:33 +08002556 bool in_use;
aliguori6d16c2f2009-01-22 16:59:11 +00002557} BounceBuffer;
2558
2559static BounceBuffer bounce;
2560
aliguoriba223c22009-01-22 16:59:16 +00002561typedef struct MapClient {
Fam Zhenge95205e2015-03-16 17:03:37 +08002562 QEMUBH *bh;
Blue Swirl72cf2d42009-09-12 07:36:22 +00002563 QLIST_ENTRY(MapClient) link;
aliguoriba223c22009-01-22 16:59:16 +00002564} MapClient;
2565
Fam Zheng38e047b2015-03-16 17:03:35 +08002566QemuMutex map_client_list_lock;
Blue Swirl72cf2d42009-09-12 07:36:22 +00002567static QLIST_HEAD(map_client_list, MapClient) map_client_list
2568 = QLIST_HEAD_INITIALIZER(map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00002569
Fam Zhenge95205e2015-03-16 17:03:37 +08002570static void cpu_unregister_map_client_do(MapClient *client)
aliguoriba223c22009-01-22 16:59:16 +00002571{
Blue Swirl72cf2d42009-09-12 07:36:22 +00002572 QLIST_REMOVE(client, link);
Anthony Liguori7267c092011-08-20 22:09:37 -05002573 g_free(client);
aliguoriba223c22009-01-22 16:59:16 +00002574}
2575
Fam Zheng33b6c2e2015-03-16 17:03:36 +08002576static void cpu_notify_map_clients_locked(void)
aliguoriba223c22009-01-22 16:59:16 +00002577{
2578 MapClient *client;
2579
Blue Swirl72cf2d42009-09-12 07:36:22 +00002580 while (!QLIST_EMPTY(&map_client_list)) {
2581 client = QLIST_FIRST(&map_client_list);
Fam Zhenge95205e2015-03-16 17:03:37 +08002582 qemu_bh_schedule(client->bh);
2583 cpu_unregister_map_client_do(client);
aliguoriba223c22009-01-22 16:59:16 +00002584 }
2585}
2586
Fam Zhenge95205e2015-03-16 17:03:37 +08002587void cpu_register_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00002588{
2589 MapClient *client = g_malloc(sizeof(*client));
2590
Fam Zheng38e047b2015-03-16 17:03:35 +08002591 qemu_mutex_lock(&map_client_list_lock);
Fam Zhenge95205e2015-03-16 17:03:37 +08002592 client->bh = bh;
bellardd0ecd2a2006-04-23 17:14:48 +00002593 QLIST_INSERT_HEAD(&map_client_list, client, link);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08002594 if (!atomic_read(&bounce.in_use)) {
2595 cpu_notify_map_clients_locked();
2596 }
Fam Zheng38e047b2015-03-16 17:03:35 +08002597 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00002598}
2599
Fam Zheng38e047b2015-03-16 17:03:35 +08002600void cpu_exec_init_all(void)
2601{
2602 qemu_mutex_init(&ram_list.mutex);
2603 memory_map_init();
2604 io_mem_init();
2605 qemu_mutex_init(&map_client_list_lock);
2606}
2607
Fam Zhenge95205e2015-03-16 17:03:37 +08002608void cpu_unregister_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00002609{
Fam Zhenge95205e2015-03-16 17:03:37 +08002610 MapClient *client;
bellardd0ecd2a2006-04-23 17:14:48 +00002611
Fam Zhenge95205e2015-03-16 17:03:37 +08002612 qemu_mutex_lock(&map_client_list_lock);
2613 QLIST_FOREACH(client, &map_client_list, link) {
2614 if (client->bh == bh) {
2615 cpu_unregister_map_client_do(client);
2616 break;
2617 }
2618 }
2619 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00002620}
2621
2622static void cpu_notify_map_clients(void)
2623{
Fam Zheng38e047b2015-03-16 17:03:35 +08002624 qemu_mutex_lock(&map_client_list_lock);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08002625 cpu_notify_map_clients_locked();
Fam Zheng38e047b2015-03-16 17:03:35 +08002626 qemu_mutex_unlock(&map_client_list_lock);
aliguori6d16c2f2009-01-22 16:59:11 +00002627}
2628
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002629bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2630{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002631 MemoryRegion *mr;
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002632 hwaddr l, xlat;
2633
Paolo Bonzini41063e12015-03-18 14:21:43 +01002634 rcu_read_lock();
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002635 while (len > 0) {
2636 l = len;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002637 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2638 if (!memory_access_is_direct(mr, is_write)) {
2639 l = memory_access_size(mr, l, addr);
2640 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002641 return false;
2642 }
2643 }
2644
2645 len -= l;
2646 addr += l;
2647 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002648 rcu_read_unlock();
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002649 return true;
2650}
2651
aliguori6d16c2f2009-01-22 16:59:11 +00002652/* Map a physical memory region into a host virtual address.
2653 * May map a subset of the requested range, given by and returned in *plen.
2654 * May return NULL if resources needed to perform the mapping are exhausted.
2655 * Use only for reads OR writes - not for read-modify-write operations.
aliguoriba223c22009-01-22 16:59:16 +00002656 * Use cpu_register_map_client() to know when retrying the map operation is
2657 * likely to succeed.
aliguori6d16c2f2009-01-22 16:59:11 +00002658 */
Avi Kivityac1970f2012-10-03 16:22:53 +02002659void *address_space_map(AddressSpace *as,
Avi Kivitya8170e52012-10-23 12:30:10 +02002660 hwaddr addr,
2661 hwaddr *plen,
Avi Kivityac1970f2012-10-03 16:22:53 +02002662 bool is_write)
aliguori6d16c2f2009-01-22 16:59:11 +00002663{
Avi Kivitya8170e52012-10-23 12:30:10 +02002664 hwaddr len = *plen;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002665 hwaddr done = 0;
2666 hwaddr l, xlat, base;
2667 MemoryRegion *mr, *this_mr;
2668 ram_addr_t raddr;
aliguori6d16c2f2009-01-22 16:59:11 +00002669
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002670 if (len == 0) {
2671 return NULL;
2672 }
aliguori6d16c2f2009-01-22 16:59:11 +00002673
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002674 l = len;
Paolo Bonzini41063e12015-03-18 14:21:43 +01002675 rcu_read_lock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002676 mr = address_space_translate(as, addr, &xlat, &l, is_write);
Paolo Bonzini41063e12015-03-18 14:21:43 +01002677
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002678 if (!memory_access_is_direct(mr, is_write)) {
Fam Zhengc2cba0f2015-03-16 17:03:33 +08002679 if (atomic_xchg(&bounce.in_use, true)) {
Paolo Bonzini41063e12015-03-18 14:21:43 +01002680 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002681 return NULL;
aliguori6d16c2f2009-01-22 16:59:11 +00002682 }
Kevin Wolfe85d9db2013-07-22 14:30:23 +02002683 /* Avoid unbounded allocations */
2684 l = MIN(l, TARGET_PAGE_SIZE);
2685 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002686 bounce.addr = addr;
2687 bounce.len = l;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002688
2689 memory_region_ref(mr);
2690 bounce.mr = mr;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002691 if (!is_write) {
Peter Maydell5c9eb022015-04-26 16:49:24 +01002692 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2693 bounce.buffer, l);
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01002694 }
aliguori6d16c2f2009-01-22 16:59:11 +00002695
Paolo Bonzini41063e12015-03-18 14:21:43 +01002696 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002697 *plen = l;
2698 return bounce.buffer;
2699 }
2700
2701 base = xlat;
2702 raddr = memory_region_get_ram_addr(mr);
2703
2704 for (;;) {
aliguori6d16c2f2009-01-22 16:59:11 +00002705 len -= l;
2706 addr += l;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002707 done += l;
2708 if (len == 0) {
2709 break;
2710 }
2711
2712 l = len;
2713 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2714 if (this_mr != mr || xlat != base + done) {
2715 break;
2716 }
aliguori6d16c2f2009-01-22 16:59:11 +00002717 }
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002718
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002719 memory_region_ref(mr);
Paolo Bonzini41063e12015-03-18 14:21:43 +01002720 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002721 *plen = done;
2722 return qemu_ram_ptr_length(raddr + base, plen);
aliguori6d16c2f2009-01-22 16:59:11 +00002723}
2724
Avi Kivityac1970f2012-10-03 16:22:53 +02002725/* Unmaps a memory region previously mapped by address_space_map().
aliguori6d16c2f2009-01-22 16:59:11 +00002726 * Will also mark the memory as dirty if is_write == 1. access_len gives
2727 * the amount of memory that was actually read or written by the caller.
2728 */
Avi Kivitya8170e52012-10-23 12:30:10 +02002729void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2730 int is_write, hwaddr access_len)
aliguori6d16c2f2009-01-22 16:59:11 +00002731{
2732 if (buffer != bounce.buffer) {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002733 MemoryRegion *mr;
2734 ram_addr_t addr1;
2735
2736 mr = qemu_ram_addr_from_host(buffer, &addr1);
2737 assert(mr != NULL);
aliguori6d16c2f2009-01-22 16:59:11 +00002738 if (is_write) {
Paolo Bonzini845b6212015-03-23 11:45:53 +01002739 invalidate_and_set_dirty(mr, addr1, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00002740 }
Jan Kiszka868bb332011-06-21 22:59:09 +02002741 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02002742 xen_invalidate_map_cache_entry(buffer);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01002743 }
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002744 memory_region_unref(mr);
aliguori6d16c2f2009-01-22 16:59:11 +00002745 return;
2746 }
2747 if (is_write) {
Peter Maydell5c9eb022015-04-26 16:49:24 +01002748 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2749 bounce.buffer, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00002750 }
Herve Poussineauf8a83242010-01-24 21:23:56 +00002751 qemu_vfree(bounce.buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00002752 bounce.buffer = NULL;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002753 memory_region_unref(bounce.mr);
Fam Zhengc2cba0f2015-03-16 17:03:33 +08002754 atomic_mb_set(&bounce.in_use, false);
aliguoriba223c22009-01-22 16:59:16 +00002755 cpu_notify_map_clients();
aliguori6d16c2f2009-01-22 16:59:11 +00002756}
bellardd0ecd2a2006-04-23 17:14:48 +00002757
Avi Kivitya8170e52012-10-23 12:30:10 +02002758void *cpu_physical_memory_map(hwaddr addr,
2759 hwaddr *plen,
Avi Kivityac1970f2012-10-03 16:22:53 +02002760 int is_write)
2761{
2762 return address_space_map(&address_space_memory, addr, plen, is_write);
2763}
2764
Avi Kivitya8170e52012-10-23 12:30:10 +02002765void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2766 int is_write, hwaddr access_len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002767{
2768 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2769}
2770
bellard8df1cd02005-01-28 22:37:22 +00002771/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01002772static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
2773 MemTxAttrs attrs,
2774 MemTxResult *result,
2775 enum device_endian endian)
bellard8df1cd02005-01-28 22:37:22 +00002776{
bellard8df1cd02005-01-28 22:37:22 +00002777 uint8_t *ptr;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002778 uint64_t val;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002779 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002780 hwaddr l = 4;
2781 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01002782 MemTxResult r;
Jan Kiszka4840f102015-06-18 18:47:22 +02002783 bool release_lock = false;
bellard8df1cd02005-01-28 22:37:22 +00002784
Paolo Bonzini41063e12015-03-18 14:21:43 +01002785 rcu_read_lock();
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002786 mr = address_space_translate(as, addr, &addr1, &l, false);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002787 if (l < 4 || !memory_access_is_direct(mr, false)) {
Jan Kiszka4840f102015-06-18 18:47:22 +02002788 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini125b3802015-06-18 18:47:21 +02002789
bellard8df1cd02005-01-28 22:37:22 +00002790 /* I/O case */
Peter Maydell50013112015-04-26 16:49:24 +01002791 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002792#if defined(TARGET_WORDS_BIGENDIAN)
2793 if (endian == DEVICE_LITTLE_ENDIAN) {
2794 val = bswap32(val);
2795 }
2796#else
2797 if (endian == DEVICE_BIG_ENDIAN) {
2798 val = bswap32(val);
2799 }
2800#endif
bellard8df1cd02005-01-28 22:37:22 +00002801 } else {
2802 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002803 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
Avi Kivity06ef3522012-02-13 16:11:22 +02002804 & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002805 + addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002806 switch (endian) {
2807 case DEVICE_LITTLE_ENDIAN:
2808 val = ldl_le_p(ptr);
2809 break;
2810 case DEVICE_BIG_ENDIAN:
2811 val = ldl_be_p(ptr);
2812 break;
2813 default:
2814 val = ldl_p(ptr);
2815 break;
2816 }
Peter Maydell50013112015-04-26 16:49:24 +01002817 r = MEMTX_OK;
2818 }
2819 if (result) {
2820 *result = r;
bellard8df1cd02005-01-28 22:37:22 +00002821 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002822 if (release_lock) {
2823 qemu_mutex_unlock_iothread();
2824 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002825 rcu_read_unlock();
bellard8df1cd02005-01-28 22:37:22 +00002826 return val;
2827}
2828
Peter Maydell50013112015-04-26 16:49:24 +01002829uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
2830 MemTxAttrs attrs, MemTxResult *result)
2831{
2832 return address_space_ldl_internal(as, addr, attrs, result,
2833 DEVICE_NATIVE_ENDIAN);
2834}
2835
2836uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
2837 MemTxAttrs attrs, MemTxResult *result)
2838{
2839 return address_space_ldl_internal(as, addr, attrs, result,
2840 DEVICE_LITTLE_ENDIAN);
2841}
2842
2843uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
2844 MemTxAttrs attrs, MemTxResult *result)
2845{
2846 return address_space_ldl_internal(as, addr, attrs, result,
2847 DEVICE_BIG_ENDIAN);
2848}
2849
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002850uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002851{
Peter Maydell50013112015-04-26 16:49:24 +01002852 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002853}
2854
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002855uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002856{
Peter Maydell50013112015-04-26 16:49:24 +01002857 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002858}
2859
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002860uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002861{
Peter Maydell50013112015-04-26 16:49:24 +01002862 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002863}
2864
bellard84b7b8e2005-11-28 21:19:04 +00002865/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01002866static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
2867 MemTxAttrs attrs,
2868 MemTxResult *result,
2869 enum device_endian endian)
bellard84b7b8e2005-11-28 21:19:04 +00002870{
bellard84b7b8e2005-11-28 21:19:04 +00002871 uint8_t *ptr;
2872 uint64_t val;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002873 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002874 hwaddr l = 8;
2875 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01002876 MemTxResult r;
Jan Kiszka4840f102015-06-18 18:47:22 +02002877 bool release_lock = false;
bellard84b7b8e2005-11-28 21:19:04 +00002878
Paolo Bonzini41063e12015-03-18 14:21:43 +01002879 rcu_read_lock();
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002880 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002881 false);
2882 if (l < 8 || !memory_access_is_direct(mr, false)) {
Jan Kiszka4840f102015-06-18 18:47:22 +02002883 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini125b3802015-06-18 18:47:21 +02002884
bellard84b7b8e2005-11-28 21:19:04 +00002885 /* I/O case */
Peter Maydell50013112015-04-26 16:49:24 +01002886 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
Paolo Bonzini968a5622013-05-24 17:58:37 +02002887#if defined(TARGET_WORDS_BIGENDIAN)
2888 if (endian == DEVICE_LITTLE_ENDIAN) {
2889 val = bswap64(val);
2890 }
2891#else
2892 if (endian == DEVICE_BIG_ENDIAN) {
2893 val = bswap64(val);
2894 }
2895#endif
bellard84b7b8e2005-11-28 21:19:04 +00002896 } else {
2897 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002898 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
Avi Kivity06ef3522012-02-13 16:11:22 +02002899 & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002900 + addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002901 switch (endian) {
2902 case DEVICE_LITTLE_ENDIAN:
2903 val = ldq_le_p(ptr);
2904 break;
2905 case DEVICE_BIG_ENDIAN:
2906 val = ldq_be_p(ptr);
2907 break;
2908 default:
2909 val = ldq_p(ptr);
2910 break;
2911 }
Peter Maydell50013112015-04-26 16:49:24 +01002912 r = MEMTX_OK;
2913 }
2914 if (result) {
2915 *result = r;
bellard84b7b8e2005-11-28 21:19:04 +00002916 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002917 if (release_lock) {
2918 qemu_mutex_unlock_iothread();
2919 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002920 rcu_read_unlock();
bellard84b7b8e2005-11-28 21:19:04 +00002921 return val;
2922}
2923
Peter Maydell50013112015-04-26 16:49:24 +01002924uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
2925 MemTxAttrs attrs, MemTxResult *result)
2926{
2927 return address_space_ldq_internal(as, addr, attrs, result,
2928 DEVICE_NATIVE_ENDIAN);
2929}
2930
2931uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
2932 MemTxAttrs attrs, MemTxResult *result)
2933{
2934 return address_space_ldq_internal(as, addr, attrs, result,
2935 DEVICE_LITTLE_ENDIAN);
2936}
2937
2938uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
2939 MemTxAttrs attrs, MemTxResult *result)
2940{
2941 return address_space_ldq_internal(as, addr, attrs, result,
2942 DEVICE_BIG_ENDIAN);
2943}
2944
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002945uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002946{
Peter Maydell50013112015-04-26 16:49:24 +01002947 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002948}
2949
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002950uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002951{
Peter Maydell50013112015-04-26 16:49:24 +01002952 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002953}
2954
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002955uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002956{
Peter Maydell50013112015-04-26 16:49:24 +01002957 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002958}
2959
bellardaab33092005-10-30 20:48:42 +00002960/* XXX: optimize */
Peter Maydell50013112015-04-26 16:49:24 +01002961uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
2962 MemTxAttrs attrs, MemTxResult *result)
bellardaab33092005-10-30 20:48:42 +00002963{
2964 uint8_t val;
Peter Maydell50013112015-04-26 16:49:24 +01002965 MemTxResult r;
2966
2967 r = address_space_rw(as, addr, attrs, &val, 1, 0);
2968 if (result) {
2969 *result = r;
2970 }
bellardaab33092005-10-30 20:48:42 +00002971 return val;
2972}
2973
Peter Maydell50013112015-04-26 16:49:24 +01002974uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
2975{
2976 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2977}
2978
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002979/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01002980static inline uint32_t address_space_lduw_internal(AddressSpace *as,
2981 hwaddr addr,
2982 MemTxAttrs attrs,
2983 MemTxResult *result,
2984 enum device_endian endian)
bellardaab33092005-10-30 20:48:42 +00002985{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002986 uint8_t *ptr;
2987 uint64_t val;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002988 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002989 hwaddr l = 2;
2990 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01002991 MemTxResult r;
Jan Kiszka4840f102015-06-18 18:47:22 +02002992 bool release_lock = false;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002993
Paolo Bonzini41063e12015-03-18 14:21:43 +01002994 rcu_read_lock();
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10002995 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002996 false);
2997 if (l < 2 || !memory_access_is_direct(mr, false)) {
Jan Kiszka4840f102015-06-18 18:47:22 +02002998 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini125b3802015-06-18 18:47:21 +02002999
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003000 /* I/O case */
Peter Maydell50013112015-04-26 16:49:24 +01003001 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003002#if defined(TARGET_WORDS_BIGENDIAN)
3003 if (endian == DEVICE_LITTLE_ENDIAN) {
3004 val = bswap16(val);
3005 }
3006#else
3007 if (endian == DEVICE_BIG_ENDIAN) {
3008 val = bswap16(val);
3009 }
3010#endif
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003011 } else {
3012 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003013 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
Avi Kivity06ef3522012-02-13 16:11:22 +02003014 & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003015 + addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003016 switch (endian) {
3017 case DEVICE_LITTLE_ENDIAN:
3018 val = lduw_le_p(ptr);
3019 break;
3020 case DEVICE_BIG_ENDIAN:
3021 val = lduw_be_p(ptr);
3022 break;
3023 default:
3024 val = lduw_p(ptr);
3025 break;
3026 }
Peter Maydell50013112015-04-26 16:49:24 +01003027 r = MEMTX_OK;
3028 }
3029 if (result) {
3030 *result = r;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003031 }
Jan Kiszka4840f102015-06-18 18:47:22 +02003032 if (release_lock) {
3033 qemu_mutex_unlock_iothread();
3034 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003035 rcu_read_unlock();
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003036 return val;
bellardaab33092005-10-30 20:48:42 +00003037}
3038
Peter Maydell50013112015-04-26 16:49:24 +01003039uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3040 MemTxAttrs attrs, MemTxResult *result)
3041{
3042 return address_space_lduw_internal(as, addr, attrs, result,
3043 DEVICE_NATIVE_ENDIAN);
3044}
3045
3046uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3047 MemTxAttrs attrs, MemTxResult *result)
3048{
3049 return address_space_lduw_internal(as, addr, attrs, result,
3050 DEVICE_LITTLE_ENDIAN);
3051}
3052
3053uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3054 MemTxAttrs attrs, MemTxResult *result)
3055{
3056 return address_space_lduw_internal(as, addr, attrs, result,
3057 DEVICE_BIG_ENDIAN);
3058}
3059
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10003060uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003061{
Peter Maydell50013112015-04-26 16:49:24 +01003062 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003063}
3064
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10003065uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003066{
Peter Maydell50013112015-04-26 16:49:24 +01003067 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003068}
3069
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10003070uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003071{
Peter Maydell50013112015-04-26 16:49:24 +01003072 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003073}
3074
bellard8df1cd02005-01-28 22:37:22 +00003075/* warning: addr must be aligned. The ram page is not masked as dirty
3076 and the code inside is not invalidated. It is useful if the dirty
3077 bits are used to track modified PTEs */
Peter Maydell50013112015-04-26 16:49:24 +01003078void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3079 MemTxAttrs attrs, MemTxResult *result)
bellard8df1cd02005-01-28 22:37:22 +00003080{
bellard8df1cd02005-01-28 22:37:22 +00003081 uint8_t *ptr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003082 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003083 hwaddr l = 4;
3084 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01003085 MemTxResult r;
Paolo Bonzini845b6212015-03-23 11:45:53 +01003086 uint8_t dirty_log_mask;
Jan Kiszka4840f102015-06-18 18:47:22 +02003087 bool release_lock = false;
bellard8df1cd02005-01-28 22:37:22 +00003088
Paolo Bonzini41063e12015-03-18 14:21:43 +01003089 rcu_read_lock();
Edgar E. Iglesias2198a122013-11-28 10:13:41 +01003090 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003091 true);
3092 if (l < 4 || !memory_access_is_direct(mr, true)) {
Jan Kiszka4840f102015-06-18 18:47:22 +02003093 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini125b3802015-06-18 18:47:21 +02003094
Peter Maydell50013112015-04-26 16:49:24 +01003095 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
bellard8df1cd02005-01-28 22:37:22 +00003096 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003097 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
pbrook5579c7f2009-04-11 14:47:08 +00003098 ptr = qemu_get_ram_ptr(addr1);
bellard8df1cd02005-01-28 22:37:22 +00003099 stl_p(ptr, val);
aliguori74576192008-10-06 14:02:03 +00003100
Paolo Bonzini845b6212015-03-23 11:45:53 +01003101 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3102 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
Paolo Bonzini58d27072015-03-23 11:56:01 +01003103 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
Peter Maydell50013112015-04-26 16:49:24 +01003104 r = MEMTX_OK;
3105 }
3106 if (result) {
3107 *result = r;
bellard8df1cd02005-01-28 22:37:22 +00003108 }
Jan Kiszka4840f102015-06-18 18:47:22 +02003109 if (release_lock) {
3110 qemu_mutex_unlock_iothread();
3111 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003112 rcu_read_unlock();
bellard8df1cd02005-01-28 22:37:22 +00003113}
3114
Peter Maydell50013112015-04-26 16:49:24 +01003115void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3116{
3117 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3118}
3119
bellard8df1cd02005-01-28 22:37:22 +00003120/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01003121static inline void address_space_stl_internal(AddressSpace *as,
3122 hwaddr addr, uint32_t val,
3123 MemTxAttrs attrs,
3124 MemTxResult *result,
3125 enum device_endian endian)
bellard8df1cd02005-01-28 22:37:22 +00003126{
bellard8df1cd02005-01-28 22:37:22 +00003127 uint8_t *ptr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003128 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003129 hwaddr l = 4;
3130 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01003131 MemTxResult r;
Jan Kiszka4840f102015-06-18 18:47:22 +02003132 bool release_lock = false;
bellard8df1cd02005-01-28 22:37:22 +00003133
Paolo Bonzini41063e12015-03-18 14:21:43 +01003134 rcu_read_lock();
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003135 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003136 true);
3137 if (l < 4 || !memory_access_is_direct(mr, true)) {
Jan Kiszka4840f102015-06-18 18:47:22 +02003138 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini125b3802015-06-18 18:47:21 +02003139
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003140#if defined(TARGET_WORDS_BIGENDIAN)
3141 if (endian == DEVICE_LITTLE_ENDIAN) {
3142 val = bswap32(val);
3143 }
3144#else
3145 if (endian == DEVICE_BIG_ENDIAN) {
3146 val = bswap32(val);
3147 }
3148#endif
Peter Maydell50013112015-04-26 16:49:24 +01003149 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
bellard8df1cd02005-01-28 22:37:22 +00003150 } else {
bellard8df1cd02005-01-28 22:37:22 +00003151 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003152 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
pbrook5579c7f2009-04-11 14:47:08 +00003153 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003154 switch (endian) {
3155 case DEVICE_LITTLE_ENDIAN:
3156 stl_le_p(ptr, val);
3157 break;
3158 case DEVICE_BIG_ENDIAN:
3159 stl_be_p(ptr, val);
3160 break;
3161 default:
3162 stl_p(ptr, val);
3163 break;
3164 }
Paolo Bonzini845b6212015-03-23 11:45:53 +01003165 invalidate_and_set_dirty(mr, addr1, 4);
Peter Maydell50013112015-04-26 16:49:24 +01003166 r = MEMTX_OK;
bellard8df1cd02005-01-28 22:37:22 +00003167 }
Peter Maydell50013112015-04-26 16:49:24 +01003168 if (result) {
3169 *result = r;
3170 }
Jan Kiszka4840f102015-06-18 18:47:22 +02003171 if (release_lock) {
3172 qemu_mutex_unlock_iothread();
3173 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003174 rcu_read_unlock();
Peter Maydell50013112015-04-26 16:49:24 +01003175}
3176
3177void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3178 MemTxAttrs attrs, MemTxResult *result)
3179{
3180 address_space_stl_internal(as, addr, val, attrs, result,
3181 DEVICE_NATIVE_ENDIAN);
3182}
3183
3184void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3185 MemTxAttrs attrs, MemTxResult *result)
3186{
3187 address_space_stl_internal(as, addr, val, attrs, result,
3188 DEVICE_LITTLE_ENDIAN);
3189}
3190
3191void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3192 MemTxAttrs attrs, MemTxResult *result)
3193{
3194 address_space_stl_internal(as, addr, val, attrs, result,
3195 DEVICE_BIG_ENDIAN);
bellard8df1cd02005-01-28 22:37:22 +00003196}
3197
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003198void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003199{
Peter Maydell50013112015-04-26 16:49:24 +01003200 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003201}
3202
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003203void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003204{
Peter Maydell50013112015-04-26 16:49:24 +01003205 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003206}
3207
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003208void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003209{
Peter Maydell50013112015-04-26 16:49:24 +01003210 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003211}
3212
bellardaab33092005-10-30 20:48:42 +00003213/* XXX: optimize */
Peter Maydell50013112015-04-26 16:49:24 +01003214void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3215 MemTxAttrs attrs, MemTxResult *result)
bellardaab33092005-10-30 20:48:42 +00003216{
3217 uint8_t v = val;
Peter Maydell50013112015-04-26 16:49:24 +01003218 MemTxResult r;
3219
3220 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3221 if (result) {
3222 *result = r;
3223 }
3224}
3225
3226void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3227{
3228 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
bellardaab33092005-10-30 20:48:42 +00003229}
3230
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003231/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01003232static inline void address_space_stw_internal(AddressSpace *as,
3233 hwaddr addr, uint32_t val,
3234 MemTxAttrs attrs,
3235 MemTxResult *result,
3236 enum device_endian endian)
bellardaab33092005-10-30 20:48:42 +00003237{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003238 uint8_t *ptr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003239 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003240 hwaddr l = 2;
3241 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01003242 MemTxResult r;
Jan Kiszka4840f102015-06-18 18:47:22 +02003243 bool release_lock = false;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003244
Paolo Bonzini41063e12015-03-18 14:21:43 +01003245 rcu_read_lock();
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003246 mr = address_space_translate(as, addr, &addr1, &l, true);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003247 if (l < 2 || !memory_access_is_direct(mr, true)) {
Jan Kiszka4840f102015-06-18 18:47:22 +02003248 release_lock |= prepare_mmio_access(mr);
Paolo Bonzini125b3802015-06-18 18:47:21 +02003249
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003250#if defined(TARGET_WORDS_BIGENDIAN)
3251 if (endian == DEVICE_LITTLE_ENDIAN) {
3252 val = bswap16(val);
3253 }
3254#else
3255 if (endian == DEVICE_BIG_ENDIAN) {
3256 val = bswap16(val);
3257 }
3258#endif
Peter Maydell50013112015-04-26 16:49:24 +01003259 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003260 } else {
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003261 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003262 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003263 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003264 switch (endian) {
3265 case DEVICE_LITTLE_ENDIAN:
3266 stw_le_p(ptr, val);
3267 break;
3268 case DEVICE_BIG_ENDIAN:
3269 stw_be_p(ptr, val);
3270 break;
3271 default:
3272 stw_p(ptr, val);
3273 break;
3274 }
Paolo Bonzini845b6212015-03-23 11:45:53 +01003275 invalidate_and_set_dirty(mr, addr1, 2);
Peter Maydell50013112015-04-26 16:49:24 +01003276 r = MEMTX_OK;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003277 }
Peter Maydell50013112015-04-26 16:49:24 +01003278 if (result) {
3279 *result = r;
3280 }
Jan Kiszka4840f102015-06-18 18:47:22 +02003281 if (release_lock) {
3282 qemu_mutex_unlock_iothread();
3283 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003284 rcu_read_unlock();
Peter Maydell50013112015-04-26 16:49:24 +01003285}
3286
3287void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3288 MemTxAttrs attrs, MemTxResult *result)
3289{
3290 address_space_stw_internal(as, addr, val, attrs, result,
3291 DEVICE_NATIVE_ENDIAN);
3292}
3293
3294void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3295 MemTxAttrs attrs, MemTxResult *result)
3296{
3297 address_space_stw_internal(as, addr, val, attrs, result,
3298 DEVICE_LITTLE_ENDIAN);
3299}
3300
3301void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3302 MemTxAttrs attrs, MemTxResult *result)
3303{
3304 address_space_stw_internal(as, addr, val, attrs, result,
3305 DEVICE_BIG_ENDIAN);
bellardaab33092005-10-30 20:48:42 +00003306}
3307
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003308void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003309{
Peter Maydell50013112015-04-26 16:49:24 +01003310 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003311}
3312
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003313void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003314{
Peter Maydell50013112015-04-26 16:49:24 +01003315 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003316}
3317
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003318void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003319{
Peter Maydell50013112015-04-26 16:49:24 +01003320 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003321}
3322
bellardaab33092005-10-30 20:48:42 +00003323/* XXX: optimize */
Peter Maydell50013112015-04-26 16:49:24 +01003324void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3325 MemTxAttrs attrs, MemTxResult *result)
3326{
3327 MemTxResult r;
3328 val = tswap64(val);
3329 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3330 if (result) {
3331 *result = r;
3332 }
3333}
3334
3335void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3336 MemTxAttrs attrs, MemTxResult *result)
3337{
3338 MemTxResult r;
3339 val = cpu_to_le64(val);
3340 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3341 if (result) {
3342 *result = r;
3343 }
3344}
3345void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3346 MemTxAttrs attrs, MemTxResult *result)
3347{
3348 MemTxResult r;
3349 val = cpu_to_be64(val);
3350 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3351 if (result) {
3352 *result = r;
3353 }
3354}
3355
Edgar E. Iglesiasf6066042013-11-28 00:11:44 +01003356void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
bellardaab33092005-10-30 20:48:42 +00003357{
Peter Maydell50013112015-04-26 16:49:24 +01003358 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
bellardaab33092005-10-30 20:48:42 +00003359}
3360
Edgar E. Iglesiasf6066042013-11-28 00:11:44 +01003361void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003362{
Peter Maydell50013112015-04-26 16:49:24 +01003363 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003364}
3365
Edgar E. Iglesiasf6066042013-11-28 00:11:44 +01003366void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003367{
Peter Maydell50013112015-04-26 16:49:24 +01003368 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003369}
3370
aliguori5e2972f2009-03-28 17:51:36 +00003371/* virtual memory access for debug (includes writing to ROM) */
Andreas Färberf17ec442013-06-29 19:40:58 +02003372int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
bellardb448f2f2004-02-25 23:24:04 +00003373 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00003374{
3375 int l;
Avi Kivitya8170e52012-10-23 12:30:10 +02003376 hwaddr phys_addr;
j_mayer9b3c35e2007-04-07 11:21:28 +00003377 target_ulong page;
bellard13eb76e2004-01-24 15:23:36 +00003378
3379 while (len > 0) {
3380 page = addr & TARGET_PAGE_MASK;
Andreas Färberf17ec442013-06-29 19:40:58 +02003381 phys_addr = cpu_get_phys_page_debug(cpu, page);
bellard13eb76e2004-01-24 15:23:36 +00003382 /* if no physical page mapped, return an error */
3383 if (phys_addr == -1)
3384 return -1;
3385 l = (page + TARGET_PAGE_SIZE) - addr;
3386 if (l > len)
3387 l = len;
aliguori5e2972f2009-03-28 17:51:36 +00003388 phys_addr += (addr & ~TARGET_PAGE_MASK);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003389 if (is_write) {
3390 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
3391 } else {
Peter Maydell5c9eb022015-04-26 16:49:24 +01003392 address_space_rw(cpu->as, phys_addr, MEMTXATTRS_UNSPECIFIED,
3393 buf, l, 0);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003394 }
bellard13eb76e2004-01-24 15:23:36 +00003395 len -= l;
3396 buf += l;
3397 addr += l;
3398 }
3399 return 0;
3400}
Paul Brooka68fe892010-03-01 00:08:59 +00003401#endif
bellard13eb76e2004-01-24 15:23:36 +00003402
Blue Swirl8e4a4242013-01-06 18:30:17 +00003403/*
3404 * A helper function for the _utterly broken_ virtio device model to find out if
3405 * it's running on a big endian machine. Don't do this at home kids!
3406 */
Greg Kurz98ed8ec2014-06-24 19:26:29 +02003407bool target_words_bigendian(void);
3408bool target_words_bigendian(void)
Blue Swirl8e4a4242013-01-06 18:30:17 +00003409{
3410#if defined(TARGET_WORDS_BIGENDIAN)
3411 return true;
3412#else
3413 return false;
3414#endif
3415}
3416
Wen Congyang76f35532012-05-07 12:04:18 +08003417#ifndef CONFIG_USER_ONLY
Avi Kivitya8170e52012-10-23 12:30:10 +02003418bool cpu_physical_memory_is_io(hwaddr phys_addr)
Wen Congyang76f35532012-05-07 12:04:18 +08003419{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003420 MemoryRegion*mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003421 hwaddr l = 1;
Paolo Bonzini41063e12015-03-18 14:21:43 +01003422 bool res;
Wen Congyang76f35532012-05-07 12:04:18 +08003423
Paolo Bonzini41063e12015-03-18 14:21:43 +01003424 rcu_read_lock();
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003425 mr = address_space_translate(&address_space_memory,
3426 phys_addr, &phys_addr, &l, false);
Wen Congyang76f35532012-05-07 12:04:18 +08003427
Paolo Bonzini41063e12015-03-18 14:21:43 +01003428 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3429 rcu_read_unlock();
3430 return res;
Wen Congyang76f35532012-05-07 12:04:18 +08003431}
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003432
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003433int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003434{
3435 RAMBlock *block;
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003436 int ret = 0;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003437
Mike Day0dc3f442013-09-05 14:41:35 -04003438 rcu_read_lock();
3439 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003440 ret = func(block->idstr, block->host, block->offset,
3441 block->used_length, opaque);
3442 if (ret) {
3443 break;
3444 }
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003445 }
Mike Day0dc3f442013-09-05 14:41:35 -04003446 rcu_read_unlock();
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003447 return ret;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003448}
Peter Maydellec3f8c92013-06-27 20:53:38 +01003449#endif