blob: 42a5a7c8f0275746619311725aad4505549ee807 [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
Blue Swirl5b6dd862012-12-02 16:04:43 +00002 * Virtual page mapping
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
Peter Maydell7b31bbc2016-01-26 18:16:56 +000019#include "qemu/osdep.h"
Markus Armbrusterda34e652016-03-14 09:01:28 +010020#include "qapi/error.h"
Stefan Weil777872e2014-02-23 18:02:08 +010021#ifndef _WIN32
bellardd5a8f072004-09-29 21:15:28 +000022#endif
bellard54936002003-05-13 00:25:15 +000023
Veronia Bahaaf348b6d2016-03-20 19:16:19 +020024#include "qemu/cutils.h"
bellard6180a182003-09-30 21:04:53 +000025#include "cpu.h"
Paolo Bonzini63c91552016-03-15 13:18:37 +010026#include "exec/exec-all.h"
Juan Quintela51180422017-04-24 20:50:19 +020027#include "exec/target_page.h"
bellardb67d9a52008-05-23 09:57:34 +000028#include "tcg.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020029#include "hw/qdev-core.h"
Michael S. Tsirkin4485bd22015-03-11 07:56:34 +010030#if !defined(CONFIG_USER_ONLY)
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +020031#include "hw/boards.h"
Paolo Bonzini33c11872016-03-15 16:58:45 +010032#include "hw/xen/xen.h"
Michael S. Tsirkin4485bd22015-03-11 07:56:34 +010033#endif
Paolo Bonzini9c17d612012-12-17 18:20:04 +010034#include "sysemu/kvm.h"
Markus Armbruster2ff3de62013-07-04 15:09:22 +020035#include "sysemu/sysemu.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010036#include "qemu/timer.h"
37#include "qemu/config-file.h"
Andreas Färber75a34032013-09-02 16:57:02 +020038#include "qemu/error-report.h"
pbrook53a59602006-03-25 19:31:22 +000039#if defined(CONFIG_USER_ONLY)
Markus Armbrustera9c94272016-06-22 19:11:19 +020040#include "qemu.h"
Jun Nakajima432d2682010-08-31 16:41:25 +010041#else /* !CONFIG_USER_ONLY */
Paolo Bonzini741da0d2014-06-27 08:40:04 +020042#include "hw/hw.h"
43#include "exec/memory.h"
Paolo Bonzinidf43d492016-03-16 10:24:54 +010044#include "exec/ioport.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020045#include "sysemu/dma.h"
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +110046#include "sysemu/numa.h"
Christian Borntraeger79ca7a12017-03-07 15:19:08 +010047#include "sysemu/hw_accel.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020048#include "exec/address-spaces.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010049#include "sysemu/xen-mapcache.h"
Daniel P. Berrange0ab8ed12017-01-25 16:14:15 +000050#include "trace-root.h"
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +000051
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +000052#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
53#include <fcntl.h>
54#include <linux/falloc.h>
55#endif
56
pbrook53a59602006-03-25 19:31:22 +000057#endif
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +010058#include "exec/cpu-all.h"
Mike Day0dc3f442013-09-05 14:41:35 -040059#include "qemu/rcu_queue.h"
Jan Kiszka4840f102015-06-18 18:47:22 +020060#include "qemu/main-loop.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000061#include "translate-all.h"
Pavel Dovgalyuk76159362015-09-17 19:25:07 +030062#include "sysemu/replay.h"
Blue Swirl0cac1b62012-04-09 16:50:52 +000063
Paolo Bonzini022c62c2012-12-17 18:19:49 +010064#include "exec/memory-internal.h"
Juan Quintela220c3eb2013-10-14 17:13:59 +020065#include "exec/ram_addr.h"
Paolo Bonzini508127e2016-01-07 16:55:28 +030066#include "exec/log.h"
Avi Kivity67d95c12011-12-15 15:25:22 +020067
Bharata B Rao9dfeca72016-05-12 09:18:12 +053068#include "migration/vmstate.h"
69
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020070#include "qemu/range.h"
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +030071#ifndef _WIN32
72#include "qemu/mmap-alloc.h"
73#endif
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020074
Peter Xube9b23c2017-05-12 12:17:41 +080075#include "monitor/monitor.h"
76
blueswir1db7b5422007-05-26 17:36:03 +000077//#define DEBUG_SUBPAGE
ths1196be32007-03-17 15:17:58 +000078
pbrook99773bd2006-04-16 15:14:59 +000079#if !defined(CONFIG_USER_ONLY)
Mike Day0dc3f442013-09-05 14:41:35 -040080/* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
81 * are protected by the ramlist lock.
82 */
Mike Day0d53d9f2015-01-21 13:45:24 +010083RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
Avi Kivity62152b82011-07-26 14:26:14 +030084
85static MemoryRegion *system_memory;
Avi Kivity309cb472011-08-08 16:09:03 +030086static MemoryRegion *system_io;
Avi Kivity62152b82011-07-26 14:26:14 +030087
Avi Kivityf6790af2012-10-02 20:13:51 +020088AddressSpace address_space_io;
89AddressSpace address_space_memory;
Avi Kivity2673a5d2012-10-02 18:49:28 +020090
Paolo Bonzini0844e002013-05-24 14:37:28 +020091MemoryRegion io_mem_rom, io_mem_notdirty;
Jan Kiszkaacc9d802013-05-26 21:55:37 +020092static MemoryRegion io_mem_unassigned;
Avi Kivity0e0df1e2012-01-02 00:32:15 +020093
Paolo Bonzini7bd4f432014-05-14 17:43:22 +080094/* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
95#define RAM_PREALLOC (1 << 0)
96
Paolo Bonzinidbcb8982014-06-10 19:15:24 +080097/* RAM is mmap-ed with MAP_SHARED */
98#define RAM_SHARED (1 << 1)
99
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +0200100/* Only a portion of RAM (used_length) is actually used, and migrated.
101 * This used_length size can change across reboots.
102 */
103#define RAM_RESIZEABLE (1 << 2)
104
pbrooke2eef172008-06-08 01:09:01 +0000105#endif
bellard9fa3e852004-01-04 18:06:42 +0000106
Peter Maydell20bccb82016-10-24 16:26:49 +0100107#ifdef TARGET_PAGE_BITS_VARY
108int target_page_bits;
109bool target_page_bits_decided;
110#endif
111
Andreas Färberbdc44642013-06-24 23:50:24 +0200112struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
bellard6a00d602005-11-21 23:25:50 +0000113/* current CPU in the current thread. It is only valid inside
114 cpu_exec() */
Paolo Bonzinif240eb62015-08-26 00:17:58 +0200115__thread CPUState *current_cpu;
pbrook2e70f6e2008-06-29 01:03:05 +0000116/* 0 = Do not count executed instructions.
thsbf20dc02008-06-30 17:22:19 +0000117 1 = Precise instruction counting.
pbrook2e70f6e2008-06-29 01:03:05 +0000118 2 = Adaptive rate instruction counting. */
Paolo Bonzini5708fc62012-11-26 15:36:40 +0100119int use_icount;
bellard6a00d602005-11-21 23:25:50 +0000120
Peter Maydell20bccb82016-10-24 16:26:49 +0100121bool set_preferred_target_page_bits(int bits)
122{
123 /* The target page size is the lowest common denominator for all
124 * the CPUs in the system, so we can only make it smaller, never
125 * larger. And we can't make it smaller once we've committed to
126 * a particular size.
127 */
128#ifdef TARGET_PAGE_BITS_VARY
129 assert(bits >= TARGET_PAGE_BITS_MIN);
130 if (target_page_bits == 0 || target_page_bits > bits) {
131 if (target_page_bits_decided) {
132 return false;
133 }
134 target_page_bits = bits;
135 }
136#endif
137 return true;
138}
139
pbrooke2eef172008-06-08 01:09:01 +0000140#if !defined(CONFIG_USER_ONLY)
Avi Kivity4346ae32012-02-10 17:00:01 +0200141
Peter Maydell20bccb82016-10-24 16:26:49 +0100142static void finalize_target_page_bits(void)
143{
144#ifdef TARGET_PAGE_BITS_VARY
145 if (target_page_bits == 0) {
146 target_page_bits = TARGET_PAGE_BITS_MIN;
147 }
148 target_page_bits_decided = true;
149#endif
150}
151
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200152typedef struct PhysPageEntry PhysPageEntry;
153
154struct PhysPageEntry {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200155 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200156 uint32_t skip : 6;
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200157 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200158 uint32_t ptr : 26;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200159};
160
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200161#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
162
Paolo Bonzini03f49952013-11-07 17:14:36 +0100163/* Size of the L2 (and L3, etc) page tables. */
Paolo Bonzini57271d62013-11-07 17:14:37 +0100164#define ADDR_SPACE_BITS 64
Paolo Bonzini03f49952013-11-07 17:14:36 +0100165
Michael S. Tsirkin026736c2013-11-13 20:13:03 +0200166#define P_L2_BITS 9
Paolo Bonzini03f49952013-11-07 17:14:36 +0100167#define P_L2_SIZE (1 << P_L2_BITS)
168
169#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
170
171typedef PhysPageEntry Node[P_L2_SIZE];
Paolo Bonzini0475d942013-05-29 12:28:21 +0200172
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200173typedef struct PhysPageMap {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100174 struct rcu_head rcu;
175
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200176 unsigned sections_nb;
177 unsigned sections_nb_alloc;
178 unsigned nodes_nb;
179 unsigned nodes_nb_alloc;
180 Node *nodes;
181 MemoryRegionSection *sections;
182} PhysPageMap;
183
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200184struct AddressSpaceDispatch {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100185 struct rcu_head rcu;
186
Fam Zheng729633c2016-03-01 14:18:24 +0800187 MemoryRegionSection *mru_section;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200188 /* This is a multi-level map on the physical address space.
189 * The bottom level has pointers to MemoryRegionSections.
190 */
191 PhysPageEntry phys_map;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200192 PhysPageMap map;
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200193 AddressSpace *as;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200194};
195
Jan Kiszka90260c62013-05-26 21:46:51 +0200196#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
197typedef struct subpage_t {
198 MemoryRegion iomem;
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200199 AddressSpace *as;
Jan Kiszka90260c62013-05-26 21:46:51 +0200200 hwaddr base;
Vijaya Kumar K2615fab2016-10-24 16:26:49 +0100201 uint16_t sub_section[];
Jan Kiszka90260c62013-05-26 21:46:51 +0200202} subpage_t;
203
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200204#define PHYS_SECTION_UNASSIGNED 0
205#define PHYS_SECTION_NOTDIRTY 1
206#define PHYS_SECTION_ROM 2
207#define PHYS_SECTION_WATCH 3
Avi Kivity5312bd82012-02-12 18:32:55 +0200208
pbrooke2eef172008-06-08 01:09:01 +0000209static void io_mem_init(void);
Avi Kivity62152b82011-07-26 14:26:14 +0300210static void memory_map_init(void);
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000211static void tcg_commit(MemoryListener *listener);
pbrooke2eef172008-06-08 01:09:01 +0000212
Avi Kivity1ec9b902012-01-02 12:47:48 +0200213static MemoryRegion io_mem_watch;
Peter Maydell32857f42015-10-01 15:29:50 +0100214
215/**
216 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
217 * @cpu: the CPU whose AddressSpace this is
218 * @as: the AddressSpace itself
219 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
220 * @tcg_as_listener: listener for tracking changes to the AddressSpace
221 */
222struct CPUAddressSpace {
223 CPUState *cpu;
224 AddressSpace *as;
225 struct AddressSpaceDispatch *memory_dispatch;
226 MemoryListener tcg_as_listener;
227};
228
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200229struct DirtyBitmapSnapshot {
230 ram_addr_t start;
231 ram_addr_t end;
232 unsigned long dirty[];
233};
234
pbrook6658ffb2007-03-16 23:58:11 +0000235#endif
bellard54936002003-05-13 00:25:15 +0000236
Paul Brook6d9a1302010-02-28 23:55:53 +0000237#if !defined(CONFIG_USER_ONLY)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200238
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200239static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200240{
Peter Lieven101420b2016-07-15 12:03:50 +0200241 static unsigned alloc_hint = 16;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200242 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
Peter Lieven101420b2016-07-15 12:03:50 +0200243 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200244 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
245 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
Peter Lieven101420b2016-07-15 12:03:50 +0200246 alloc_hint = map->nodes_nb_alloc;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200247 }
248}
249
Paolo Bonzinidb946042015-05-21 15:12:29 +0200250static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200251{
252 unsigned i;
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200253 uint32_t ret;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200254 PhysPageEntry e;
255 PhysPageEntry *p;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200256
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200257 ret = map->nodes_nb++;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200258 p = map->nodes[ret];
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200259 assert(ret != PHYS_MAP_NODE_NIL);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200260 assert(ret != map->nodes_nb_alloc);
Paolo Bonzinidb946042015-05-21 15:12:29 +0200261
262 e.skip = leaf ? 0 : 1;
263 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100264 for (i = 0; i < P_L2_SIZE; ++i) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200265 memcpy(&p[i], &e, sizeof(e));
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200266 }
Avi Kivityf7bf5462012-02-13 20:12:05 +0200267 return ret;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200268}
269
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200270static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
271 hwaddr *index, hwaddr *nb, uint16_t leaf,
Avi Kivity29990972012-02-13 20:21:20 +0200272 int level)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200273{
274 PhysPageEntry *p;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100275 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200276
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200277 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200278 lp->ptr = phys_map_node_alloc(map, level == 0);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200279 }
Paolo Bonzinidb946042015-05-21 15:12:29 +0200280 p = map->nodes[lp->ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100281 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf7bf5462012-02-13 20:12:05 +0200282
Paolo Bonzini03f49952013-11-07 17:14:36 +0100283 while (*nb && lp < &p[P_L2_SIZE]) {
Avi Kivity07f07b32012-02-13 20:45:32 +0200284 if ((*index & (step - 1)) == 0 && *nb >= step) {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200285 lp->skip = 0;
Avi Kivityc19e8802012-02-13 20:25:31 +0200286 lp->ptr = leaf;
Avi Kivity07f07b32012-02-13 20:45:32 +0200287 *index += step;
288 *nb -= step;
Avi Kivity29990972012-02-13 20:21:20 +0200289 } else {
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200290 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
Avi Kivity29990972012-02-13 20:21:20 +0200291 }
292 ++lp;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200293 }
294}
295
Avi Kivityac1970f2012-10-03 16:22:53 +0200296static void phys_page_set(AddressSpaceDispatch *d,
Avi Kivitya8170e52012-10-23 12:30:10 +0200297 hwaddr index, hwaddr nb,
Avi Kivity29990972012-02-13 20:21:20 +0200298 uint16_t leaf)
bellard92e873b2004-05-21 14:52:29 +0000299{
Avi Kivity29990972012-02-13 20:21:20 +0200300 /* Wildly overreserve - it doesn't matter much. */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200301 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
bellard92e873b2004-05-21 14:52:29 +0000302
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200303 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
bellard92e873b2004-05-21 14:52:29 +0000304}
305
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200306/* Compact a non leaf page entry. Simply detect that the entry has a single child,
307 * and update our entry so we can skip it and go directly to the destination.
308 */
Marc-André Lureauefee6782016-09-28 16:37:20 +0400309static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200310{
311 unsigned valid_ptr = P_L2_SIZE;
312 int valid = 0;
313 PhysPageEntry *p;
314 int i;
315
316 if (lp->ptr == PHYS_MAP_NODE_NIL) {
317 return;
318 }
319
320 p = nodes[lp->ptr];
321 for (i = 0; i < P_L2_SIZE; i++) {
322 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
323 continue;
324 }
325
326 valid_ptr = i;
327 valid++;
328 if (p[i].skip) {
Marc-André Lureauefee6782016-09-28 16:37:20 +0400329 phys_page_compact(&p[i], nodes);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200330 }
331 }
332
333 /* We can only compress if there's only one child. */
334 if (valid != 1) {
335 return;
336 }
337
338 assert(valid_ptr < P_L2_SIZE);
339
340 /* Don't compress if it won't fit in the # of bits we have. */
341 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
342 return;
343 }
344
345 lp->ptr = p[valid_ptr].ptr;
346 if (!p[valid_ptr].skip) {
347 /* If our only child is a leaf, make this a leaf. */
348 /* By design, we should have made this node a leaf to begin with so we
349 * should never reach here.
350 * But since it's so simple to handle this, let's do it just in case we
351 * change this rule.
352 */
353 lp->skip = 0;
354 } else {
355 lp->skip += p[valid_ptr].skip;
356 }
357}
358
359static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
360{
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200361 if (d->phys_map.skip) {
Marc-André Lureauefee6782016-09-28 16:37:20 +0400362 phys_page_compact(&d->phys_map, d->map.nodes);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200363 }
364}
365
Fam Zheng29cb5332016-03-01 14:18:23 +0800366static inline bool section_covers_addr(const MemoryRegionSection *section,
367 hwaddr addr)
368{
369 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
370 * the section must cover the entire address space.
371 */
Richard Henderson258dfaa2016-06-29 15:48:03 -0700372 return int128_gethi(section->size) ||
Fam Zheng29cb5332016-03-01 14:18:23 +0800373 range_covers_byte(section->offset_within_address_space,
Richard Henderson258dfaa2016-06-29 15:48:03 -0700374 int128_getlo(section->size), addr);
Fam Zheng29cb5332016-03-01 14:18:23 +0800375}
376
Peter Xu003a0cf2017-05-15 16:50:57 +0800377static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
bellard92e873b2004-05-21 14:52:29 +0000378{
Peter Xu003a0cf2017-05-15 16:50:57 +0800379 PhysPageEntry lp = d->phys_map, *p;
380 Node *nodes = d->map.nodes;
381 MemoryRegionSection *sections = d->map.sections;
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200382 hwaddr index = addr >> TARGET_PAGE_BITS;
Avi Kivity31ab2b42012-02-13 16:44:19 +0200383 int i;
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200384
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200385 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
Avi Kivityc19e8802012-02-13 20:25:31 +0200386 if (lp.ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200387 return &sections[PHYS_SECTION_UNASSIGNED];
Avi Kivity31ab2b42012-02-13 16:44:19 +0200388 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200389 p = nodes[lp.ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100390 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200391 }
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200392
Fam Zheng29cb5332016-03-01 14:18:23 +0800393 if (section_covers_addr(&sections[lp.ptr], addr)) {
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200394 return &sections[lp.ptr];
395 } else {
396 return &sections[PHYS_SECTION_UNASSIGNED];
397 }
Avi Kivityf3705d52012-03-08 16:16:34 +0200398}
399
Blue Swirle5548612012-04-21 13:08:33 +0000400bool memory_region_is_unassigned(MemoryRegion *mr)
401{
Paolo Bonzini2a8e7492013-05-24 14:34:08 +0200402 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
Blue Swirle5548612012-04-21 13:08:33 +0000403 && mr != &io_mem_watch;
404}
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200405
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100406/* Called from RCU critical section */
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200407static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
Jan Kiszka90260c62013-05-26 21:46:51 +0200408 hwaddr addr,
409 bool resolve_subpage)
Jan Kiszka9f029602013-05-06 16:48:02 +0200410{
Fam Zheng729633c2016-03-01 14:18:24 +0800411 MemoryRegionSection *section = atomic_read(&d->mru_section);
Jan Kiszka90260c62013-05-26 21:46:51 +0200412 subpage_t *subpage;
Fam Zheng729633c2016-03-01 14:18:24 +0800413 bool update;
Jan Kiszka90260c62013-05-26 21:46:51 +0200414
Fam Zheng729633c2016-03-01 14:18:24 +0800415 if (section && section != &d->map.sections[PHYS_SECTION_UNASSIGNED] &&
416 section_covers_addr(section, addr)) {
417 update = false;
418 } else {
Peter Xu003a0cf2017-05-15 16:50:57 +0800419 section = phys_page_find(d, addr);
Fam Zheng729633c2016-03-01 14:18:24 +0800420 update = true;
421 }
Jan Kiszka90260c62013-05-26 21:46:51 +0200422 if (resolve_subpage && section->mr->subpage) {
423 subpage = container_of(section->mr, subpage_t, iomem);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200424 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
Jan Kiszka90260c62013-05-26 21:46:51 +0200425 }
Fam Zheng729633c2016-03-01 14:18:24 +0800426 if (update) {
427 atomic_set(&d->mru_section, section);
428 }
Jan Kiszka90260c62013-05-26 21:46:51 +0200429 return section;
Jan Kiszka9f029602013-05-06 16:48:02 +0200430}
431
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100432/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200433static MemoryRegionSection *
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200434address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
Jan Kiszka90260c62013-05-26 21:46:51 +0200435 hwaddr *plen, bool resolve_subpage)
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200436{
437 MemoryRegionSection *section;
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200438 MemoryRegion *mr;
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100439 Int128 diff;
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200440
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200441 section = address_space_lookup_region(d, addr, resolve_subpage);
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200442 /* Compute offset within MemoryRegionSection */
443 addr -= section->offset_within_address_space;
444
445 /* Compute offset within MemoryRegion */
446 *xlat = addr + section->offset_within_region;
447
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200448 mr = section->mr;
Paolo Bonzinib242e0e2015-07-04 00:24:51 +0200449
450 /* MMIO registers can be expected to perform full-width accesses based only
451 * on their address, without considering adjacent registers that could
452 * decode to completely different MemoryRegions. When such registers
453 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
454 * regions overlap wildly. For this reason we cannot clamp the accesses
455 * here.
456 *
457 * If the length is small (as is the case for address_space_ldl/stl),
458 * everything works fine. If the incoming length is large, however,
459 * the caller really has to do the clamping through memory_access_size.
460 */
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200461 if (memory_region_is_ram(mr)) {
Paolo Bonzinie4a511f2015-06-17 10:36:54 +0200462 diff = int128_sub(section->size, int128_make64(addr));
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200463 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
464 }
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200465 return section;
466}
Jan Kiszka90260c62013-05-26 21:46:51 +0200467
Paolo Bonzini41063e12015-03-18 14:21:43 +0100468/* Called from RCU critical section */
Peter Xua7640402017-05-17 16:57:42 +0800469static MemoryRegionSection address_space_do_translate(AddressSpace *as,
470 hwaddr addr,
471 hwaddr *xlat,
472 hwaddr *plen,
473 bool is_write,
474 bool is_mmio)
Jan Kiszka90260c62013-05-26 21:46:51 +0200475{
Avi Kivity30951152012-10-30 13:47:46 +0200476 IOMMUTLBEntry iotlb;
477 MemoryRegionSection *section;
478 MemoryRegion *mr;
Avi Kivity30951152012-10-30 13:47:46 +0200479
480 for (;;) {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100481 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
Peter Xua7640402017-05-17 16:57:42 +0800482 section = address_space_translate_internal(d, addr, &addr, plen, is_mmio);
Avi Kivity30951152012-10-30 13:47:46 +0200483 mr = section->mr;
484
485 if (!mr->iommu_ops) {
486 break;
487 }
488
Peter Xubf55b7a2017-05-19 11:19:40 +0800489 iotlb = mr->iommu_ops->translate(mr, addr, is_write ?
490 IOMMU_WO : IOMMU_RO);
Avi Kivity30951152012-10-30 13:47:46 +0200491 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
492 | (addr & iotlb.addr_mask));
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700493 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
Avi Kivity30951152012-10-30 13:47:46 +0200494 if (!(iotlb.perm & (1 << is_write))) {
Peter Xua7640402017-05-17 16:57:42 +0800495 goto translate_fail;
Avi Kivity30951152012-10-30 13:47:46 +0200496 }
497
498 as = iotlb.target_as;
499 }
500
Peter Xua7640402017-05-17 16:57:42 +0800501 *xlat = addr;
502
503 return *section;
504
505translate_fail:
506 return (MemoryRegionSection) { .mr = &io_mem_unassigned };
507}
508
509/* Called from RCU critical section */
510IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
511 bool is_write)
512{
513 MemoryRegionSection section;
514 hwaddr xlat, plen;
515
516 /* Try to get maximum page mask during translation. */
517 plen = (hwaddr)-1;
518
519 /* This can never be MMIO. */
520 section = address_space_do_translate(as, addr, &xlat, &plen,
521 is_write, false);
522
523 /* Illegal translation */
524 if (section.mr == &io_mem_unassigned) {
525 goto iotlb_fail;
526 }
527
528 /* Convert memory region offset into address space offset */
529 xlat += section.offset_within_address_space -
530 section.offset_within_region;
531
532 if (plen == (hwaddr)-1) {
533 /*
534 * We use default page size here. Logically it only happens
535 * for identity mappings.
536 */
537 plen = TARGET_PAGE_SIZE;
538 }
539
540 /* Convert to address mask */
541 plen -= 1;
542
543 return (IOMMUTLBEntry) {
544 .target_as = section.address_space,
545 .iova = addr & ~plen,
546 .translated_addr = xlat & ~plen,
547 .addr_mask = plen,
548 /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
549 .perm = IOMMU_RW,
550 };
551
552iotlb_fail:
553 return (IOMMUTLBEntry) {0};
554}
555
556/* Called from RCU critical section */
557MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
558 hwaddr *xlat, hwaddr *plen,
559 bool is_write)
560{
561 MemoryRegion *mr;
562 MemoryRegionSection section;
563
564 /* This can be MMIO, so setup MMIO bit. */
565 section = address_space_do_translate(as, addr, xlat, plen, is_write, true);
566 mr = section.mr;
567
Alexey Kardashevskiyfe680d02014-05-07 13:40:39 +0000568 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100569 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700570 *plen = MIN(page, *plen);
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100571 }
572
Avi Kivity30951152012-10-30 13:47:46 +0200573 return mr;
Jan Kiszka90260c62013-05-26 21:46:51 +0200574}
575
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100576/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200577MemoryRegionSection *
Peter Maydelld7898cd2016-01-21 14:15:05 +0000578address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +0200579 hwaddr *xlat, hwaddr *plen)
Jan Kiszka90260c62013-05-26 21:46:51 +0200580{
Avi Kivity30951152012-10-30 13:47:46 +0200581 MemoryRegionSection *section;
Alex Bennéef35e44e2016-10-21 16:34:18 +0100582 AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
Peter Maydelld7898cd2016-01-21 14:15:05 +0000583
584 section = address_space_translate_internal(d, addr, xlat, plen, false);
Avi Kivity30951152012-10-30 13:47:46 +0200585
586 assert(!section->mr->iommu_ops);
587 return section;
Jan Kiszka90260c62013-05-26 21:46:51 +0200588}
bellard9fa3e852004-01-04 18:06:42 +0000589#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000590
Andreas Färberb170fce2013-01-20 20:23:22 +0100591#if !defined(CONFIG_USER_ONLY)
pbrook9656f322008-07-01 20:01:19 +0000592
Juan Quintelae59fb372009-09-29 22:48:21 +0200593static int cpu_common_post_load(void *opaque, int version_id)
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200594{
Andreas Färber259186a2013-01-17 18:51:17 +0100595 CPUState *cpu = opaque;
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200596
aurel323098dba2009-03-07 21:28:24 +0000597 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
598 version_id is increased. */
Andreas Färber259186a2013-01-17 18:51:17 +0100599 cpu->interrupt_request &= ~0x01;
Alex Bennéed10eb082016-11-14 14:17:28 +0000600 tlb_flush(cpu);
pbrook9656f322008-07-01 20:01:19 +0000601
602 return 0;
603}
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200604
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400605static int cpu_common_pre_load(void *opaque)
606{
607 CPUState *cpu = opaque;
608
Paolo Bonziniadee6422014-12-19 12:53:14 +0100609 cpu->exception_index = -1;
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400610
611 return 0;
612}
613
614static bool cpu_common_exception_index_needed(void *opaque)
615{
616 CPUState *cpu = opaque;
617
Paolo Bonziniadee6422014-12-19 12:53:14 +0100618 return tcg_enabled() && cpu->exception_index != -1;
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400619}
620
621static const VMStateDescription vmstate_cpu_common_exception_index = {
622 .name = "cpu_common/exception_index",
623 .version_id = 1,
624 .minimum_version_id = 1,
Juan Quintela5cd8cad2014-09-23 14:09:54 +0200625 .needed = cpu_common_exception_index_needed,
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400626 .fields = (VMStateField[]) {
627 VMSTATE_INT32(exception_index, CPUState),
628 VMSTATE_END_OF_LIST()
629 }
630};
631
Andrey Smetaninbac05aa2015-07-03 15:01:44 +0300632static bool cpu_common_crash_occurred_needed(void *opaque)
633{
634 CPUState *cpu = opaque;
635
636 return cpu->crash_occurred;
637}
638
639static const VMStateDescription vmstate_cpu_common_crash_occurred = {
640 .name = "cpu_common/crash_occurred",
641 .version_id = 1,
642 .minimum_version_id = 1,
643 .needed = cpu_common_crash_occurred_needed,
644 .fields = (VMStateField[]) {
645 VMSTATE_BOOL(crash_occurred, CPUState),
646 VMSTATE_END_OF_LIST()
647 }
648};
649
Andreas Färber1a1562f2013-06-17 04:09:11 +0200650const VMStateDescription vmstate_cpu_common = {
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200651 .name = "cpu_common",
652 .version_id = 1,
653 .minimum_version_id = 1,
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400654 .pre_load = cpu_common_pre_load,
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200655 .post_load = cpu_common_post_load,
Juan Quintela35d08452014-04-16 16:01:33 +0200656 .fields = (VMStateField[]) {
Andreas Färber259186a2013-01-17 18:51:17 +0100657 VMSTATE_UINT32(halted, CPUState),
658 VMSTATE_UINT32(interrupt_request, CPUState),
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200659 VMSTATE_END_OF_LIST()
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400660 },
Juan Quintela5cd8cad2014-09-23 14:09:54 +0200661 .subsections = (const VMStateDescription*[]) {
662 &vmstate_cpu_common_exception_index,
Andrey Smetaninbac05aa2015-07-03 15:01:44 +0300663 &vmstate_cpu_common_crash_occurred,
Juan Quintela5cd8cad2014-09-23 14:09:54 +0200664 NULL
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200665 }
666};
Andreas Färber1a1562f2013-06-17 04:09:11 +0200667
pbrook9656f322008-07-01 20:01:19 +0000668#endif
669
Andreas Färber38d8f5c2012-12-17 19:47:15 +0100670CPUState *qemu_get_cpu(int index)
Glauber Costa950f1472009-06-09 12:15:18 -0400671{
Andreas Färberbdc44642013-06-24 23:50:24 +0200672 CPUState *cpu;
Glauber Costa950f1472009-06-09 12:15:18 -0400673
Andreas Färberbdc44642013-06-24 23:50:24 +0200674 CPU_FOREACH(cpu) {
Andreas Färber55e5c282012-12-17 06:18:02 +0100675 if (cpu->cpu_index == index) {
Andreas Färberbdc44642013-06-24 23:50:24 +0200676 return cpu;
Andreas Färber55e5c282012-12-17 06:18:02 +0100677 }
Glauber Costa950f1472009-06-09 12:15:18 -0400678 }
679
Andreas Färberbdc44642013-06-24 23:50:24 +0200680 return NULL;
Glauber Costa950f1472009-06-09 12:15:18 -0400681}
682
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000683#if !defined(CONFIG_USER_ONLY)
Peter Maydell56943e82016-01-21 14:15:04 +0000684void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000685{
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000686 CPUAddressSpace *newas;
687
688 /* Target code should have set num_ases before calling us */
689 assert(asidx < cpu->num_ases);
690
Peter Maydell56943e82016-01-21 14:15:04 +0000691 if (asidx == 0) {
692 /* address space 0 gets the convenience alias */
693 cpu->as = as;
694 }
695
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000696 /* KVM cannot currently support multiple address spaces. */
697 assert(asidx == 0 || !kvm_enabled());
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000698
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000699 if (!cpu->cpu_ases) {
700 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000701 }
Peter Maydell32857f42015-10-01 15:29:50 +0100702
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000703 newas = &cpu->cpu_ases[asidx];
704 newas->cpu = cpu;
705 newas->as = as;
Peter Maydell56943e82016-01-21 14:15:04 +0000706 if (tcg_enabled()) {
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000707 newas->tcg_as_listener.commit = tcg_commit;
708 memory_listener_register(&newas->tcg_as_listener, as);
Peter Maydell56943e82016-01-21 14:15:04 +0000709 }
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000710}
Peter Maydell651a5bc2016-01-21 14:15:05 +0000711
712AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
713{
714 /* Return the AddressSpace corresponding to the specified index */
715 return cpu->cpu_ases[asidx].as;
716}
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000717#endif
718
Laurent Vivier7bbc1242016-10-20 13:26:04 +0200719void cpu_exec_unrealizefn(CPUState *cpu)
Bharata B Rao1c59eb32016-05-12 09:18:11 +0530720{
Bharata B Rao9dfeca72016-05-12 09:18:12 +0530721 CPUClass *cc = CPU_GET_CLASS(cpu);
722
Paolo Bonzini267f6852016-08-28 03:45:14 +0200723 cpu_list_remove(cpu);
Bharata B Rao9dfeca72016-05-12 09:18:12 +0530724
725 if (cc->vmsd != NULL) {
726 vmstate_unregister(NULL, cc->vmsd, cpu);
727 }
728 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
729 vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
730 }
Bharata B Rao1c59eb32016-05-12 09:18:11 +0530731}
732
Laurent Vivier39e329e2016-10-20 13:26:02 +0200733void cpu_exec_initfn(CPUState *cpu)
bellardfd6ce8f2003-05-14 19:00:11 +0000734{
Peter Maydell56943e82016-01-21 14:15:04 +0000735 cpu->as = NULL;
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000736 cpu->num_ases = 0;
Peter Maydell56943e82016-01-21 14:15:04 +0000737
Eduardo Habkost291135b2015-04-27 17:00:33 -0300738#ifndef CONFIG_USER_ONLY
Eduardo Habkost291135b2015-04-27 17:00:33 -0300739 cpu->thread_id = qemu_get_thread_id();
Peter Crosthwaite6731d862016-01-21 14:15:06 +0000740
741 /* This is a softmmu CPU object, so create a property for it
742 * so users can wire up its memory. (This can't go in qom/cpu.c
743 * because that file is compiled only once for both user-mode
744 * and system builds.) The default if no link is set up is to use
745 * the system address space.
746 */
747 object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
748 (Object **)&cpu->memory,
749 qdev_prop_allow_set_link_before_realize,
750 OBJ_PROP_LINK_UNREF_ON_RELEASE,
751 &error_abort);
752 cpu->memory = system_memory;
753 object_ref(OBJECT(cpu->memory));
Eduardo Habkost291135b2015-04-27 17:00:33 -0300754#endif
Laurent Vivier39e329e2016-10-20 13:26:02 +0200755}
756
Laurent Vivierce5b1bb2016-10-20 13:26:03 +0200757void cpu_exec_realizefn(CPUState *cpu, Error **errp)
Laurent Vivier39e329e2016-10-20 13:26:02 +0200758{
759 CPUClass *cc ATTRIBUTE_UNUSED = CPU_GET_CLASS(cpu);
Eduardo Habkost291135b2015-04-27 17:00:33 -0300760
Paolo Bonzini267f6852016-08-28 03:45:14 +0200761 cpu_list_add(cpu);
Igor Mammedov1bc7e522016-07-25 11:59:19 +0200762
763#ifndef CONFIG_USER_ONLY
Andreas Färbere0d47942013-07-29 04:07:50 +0200764 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
Paolo Bonzini741da0d2014-06-27 08:40:04 +0200765 vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
Andreas Färbere0d47942013-07-29 04:07:50 +0200766 }
Andreas Färberb170fce2013-01-20 20:23:22 +0100767 if (cc->vmsd != NULL) {
Paolo Bonzini741da0d2014-06-27 08:40:04 +0200768 vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
Andreas Färberb170fce2013-01-20 20:23:22 +0100769 }
Paolo Bonzini741da0d2014-06-27 08:40:04 +0200770#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000771}
772
Andreas Färber00b941e2013-06-29 18:55:54 +0200773static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
Paul Brook94df27f2010-02-28 23:47:45 +0000774{
Peter Maydella9353fe2016-12-06 18:07:09 +0000775 /* Flush the whole TB as this will not have race conditions
776 * even if we don't have proper locking yet.
777 * Ideally we would just invalidate the TBs for the
778 * specified PC.
779 */
780 tb_flush(cpu);
Paul Brook94df27f2010-02-28 23:47:45 +0000781}
bellardd720b932004-04-25 17:57:43 +0000782
Paul Brookc527ee82010-03-01 03:31:14 +0000783#if defined(CONFIG_USER_ONLY)
Andreas Färber75a34032013-09-02 16:57:02 +0200784void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
Paul Brookc527ee82010-03-01 03:31:14 +0000785
786{
787}
788
Peter Maydell3ee887e2014-09-12 14:06:48 +0100789int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
790 int flags)
791{
792 return -ENOSYS;
793}
794
795void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
796{
797}
798
Andreas Färber75a34032013-09-02 16:57:02 +0200799int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
Paul Brookc527ee82010-03-01 03:31:14 +0000800 int flags, CPUWatchpoint **watchpoint)
801{
802 return -ENOSYS;
803}
804#else
pbrook6658ffb2007-03-16 23:58:11 +0000805/* Add a watchpoint. */
Andreas Färber75a34032013-09-02 16:57:02 +0200806int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
aliguoria1d1bb32008-11-18 20:07:32 +0000807 int flags, CPUWatchpoint **watchpoint)
pbrook6658ffb2007-03-16 23:58:11 +0000808{
aliguoric0ce9982008-11-25 22:13:57 +0000809 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +0000810
Peter Maydell05068c02014-09-12 14:06:48 +0100811 /* forbid ranges which are empty or run off the end of the address space */
Max Filippov07e28632014-09-17 22:03:36 -0700812 if (len == 0 || (addr + len - 1) < addr) {
Andreas Färber75a34032013-09-02 16:57:02 +0200813 error_report("tried to set invalid watchpoint at %"
814 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
aliguorib4051332008-11-18 20:14:20 +0000815 return -EINVAL;
816 }
Anthony Liguori7267c092011-08-20 22:09:37 -0500817 wp = g_malloc(sizeof(*wp));
pbrook6658ffb2007-03-16 23:58:11 +0000818
aliguoria1d1bb32008-11-18 20:07:32 +0000819 wp->vaddr = addr;
Peter Maydell05068c02014-09-12 14:06:48 +0100820 wp->len = len;
aliguoria1d1bb32008-11-18 20:07:32 +0000821 wp->flags = flags;
822
aliguori2dc9f412008-11-18 20:56:59 +0000823 /* keep all GDB-injected watchpoints in front */
Andreas Färberff4700b2013-08-26 18:23:18 +0200824 if (flags & BP_GDB) {
825 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
826 } else {
827 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
828 }
aliguoria1d1bb32008-11-18 20:07:32 +0000829
Andreas Färber31b030d2013-09-04 01:29:02 +0200830 tlb_flush_page(cpu, addr);
aliguoria1d1bb32008-11-18 20:07:32 +0000831
832 if (watchpoint)
833 *watchpoint = wp;
834 return 0;
pbrook6658ffb2007-03-16 23:58:11 +0000835}
836
aliguoria1d1bb32008-11-18 20:07:32 +0000837/* Remove a specific watchpoint. */
Andreas Färber75a34032013-09-02 16:57:02 +0200838int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
aliguoria1d1bb32008-11-18 20:07:32 +0000839 int flags)
pbrook6658ffb2007-03-16 23:58:11 +0000840{
aliguoria1d1bb32008-11-18 20:07:32 +0000841 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +0000842
Andreas Färberff4700b2013-08-26 18:23:18 +0200843 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +0100844 if (addr == wp->vaddr && len == wp->len
aliguori6e140f22008-11-18 20:37:55 +0000845 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
Andreas Färber75a34032013-09-02 16:57:02 +0200846 cpu_watchpoint_remove_by_ref(cpu, wp);
pbrook6658ffb2007-03-16 23:58:11 +0000847 return 0;
848 }
849 }
aliguoria1d1bb32008-11-18 20:07:32 +0000850 return -ENOENT;
pbrook6658ffb2007-03-16 23:58:11 +0000851}
852
aliguoria1d1bb32008-11-18 20:07:32 +0000853/* Remove a specific watchpoint by reference. */
Andreas Färber75a34032013-09-02 16:57:02 +0200854void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
aliguoria1d1bb32008-11-18 20:07:32 +0000855{
Andreas Färberff4700b2013-08-26 18:23:18 +0200856 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
edgar_igl7d03f822008-05-17 18:58:29 +0000857
Andreas Färber31b030d2013-09-04 01:29:02 +0200858 tlb_flush_page(cpu, watchpoint->vaddr);
aliguoria1d1bb32008-11-18 20:07:32 +0000859
Anthony Liguori7267c092011-08-20 22:09:37 -0500860 g_free(watchpoint);
edgar_igl7d03f822008-05-17 18:58:29 +0000861}
862
aliguoria1d1bb32008-11-18 20:07:32 +0000863/* Remove all matching watchpoints. */
Andreas Färber75a34032013-09-02 16:57:02 +0200864void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
aliguoria1d1bb32008-11-18 20:07:32 +0000865{
aliguoric0ce9982008-11-25 22:13:57 +0000866 CPUWatchpoint *wp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +0000867
Andreas Färberff4700b2013-08-26 18:23:18 +0200868 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
Andreas Färber75a34032013-09-02 16:57:02 +0200869 if (wp->flags & mask) {
870 cpu_watchpoint_remove_by_ref(cpu, wp);
871 }
aliguoric0ce9982008-11-25 22:13:57 +0000872 }
aliguoria1d1bb32008-11-18 20:07:32 +0000873}
Peter Maydell05068c02014-09-12 14:06:48 +0100874
875/* Return true if this watchpoint address matches the specified
876 * access (ie the address range covered by the watchpoint overlaps
877 * partially or completely with the address range covered by the
878 * access).
879 */
880static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
881 vaddr addr,
882 vaddr len)
883{
884 /* We know the lengths are non-zero, but a little caution is
885 * required to avoid errors in the case where the range ends
886 * exactly at the top of the address space and so addr + len
887 * wraps round to zero.
888 */
889 vaddr wpend = wp->vaddr + wp->len - 1;
890 vaddr addrend = addr + len - 1;
891
892 return !(addr > wpend || wp->vaddr > addrend);
893}
894
Paul Brookc527ee82010-03-01 03:31:14 +0000895#endif
aliguoria1d1bb32008-11-18 20:07:32 +0000896
897/* Add a breakpoint. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200898int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
aliguoria1d1bb32008-11-18 20:07:32 +0000899 CPUBreakpoint **breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +0000900{
aliguoric0ce9982008-11-25 22:13:57 +0000901 CPUBreakpoint *bp;
ths3b46e622007-09-17 08:09:54 +0000902
Anthony Liguori7267c092011-08-20 22:09:37 -0500903 bp = g_malloc(sizeof(*bp));
aliguoria1d1bb32008-11-18 20:07:32 +0000904
905 bp->pc = pc;
906 bp->flags = flags;
907
aliguori2dc9f412008-11-18 20:56:59 +0000908 /* keep all GDB-injected breakpoints in front */
Andreas Färber00b941e2013-06-29 18:55:54 +0200909 if (flags & BP_GDB) {
Andreas Färberf0c3c502013-08-26 21:22:53 +0200910 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
Andreas Färber00b941e2013-06-29 18:55:54 +0200911 } else {
Andreas Färberf0c3c502013-08-26 21:22:53 +0200912 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
Andreas Färber00b941e2013-06-29 18:55:54 +0200913 }
aliguoria1d1bb32008-11-18 20:07:32 +0000914
Andreas Färberf0c3c502013-08-26 21:22:53 +0200915 breakpoint_invalidate(cpu, pc);
aliguoria1d1bb32008-11-18 20:07:32 +0000916
Andreas Färber00b941e2013-06-29 18:55:54 +0200917 if (breakpoint) {
aliguoria1d1bb32008-11-18 20:07:32 +0000918 *breakpoint = bp;
Andreas Färber00b941e2013-06-29 18:55:54 +0200919 }
aliguoria1d1bb32008-11-18 20:07:32 +0000920 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +0000921}
922
923/* Remove a specific breakpoint. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200924int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
aliguoria1d1bb32008-11-18 20:07:32 +0000925{
aliguoria1d1bb32008-11-18 20:07:32 +0000926 CPUBreakpoint *bp;
927
Andreas Färberf0c3c502013-08-26 21:22:53 +0200928 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
aliguoria1d1bb32008-11-18 20:07:32 +0000929 if (bp->pc == pc && bp->flags == flags) {
Andreas Färberb3310ab2013-09-02 17:26:20 +0200930 cpu_breakpoint_remove_by_ref(cpu, bp);
bellard4c3a88a2003-07-26 12:06:08 +0000931 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +0000932 }
bellard4c3a88a2003-07-26 12:06:08 +0000933 }
aliguoria1d1bb32008-11-18 20:07:32 +0000934 return -ENOENT;
bellard4c3a88a2003-07-26 12:06:08 +0000935}
936
aliguoria1d1bb32008-11-18 20:07:32 +0000937/* Remove a specific breakpoint by reference. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200938void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +0000939{
Andreas Färberf0c3c502013-08-26 21:22:53 +0200940 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
941
942 breakpoint_invalidate(cpu, breakpoint->pc);
aliguoria1d1bb32008-11-18 20:07:32 +0000943
Anthony Liguori7267c092011-08-20 22:09:37 -0500944 g_free(breakpoint);
aliguoria1d1bb32008-11-18 20:07:32 +0000945}
946
947/* Remove all matching breakpoints. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200948void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
aliguoria1d1bb32008-11-18 20:07:32 +0000949{
aliguoric0ce9982008-11-25 22:13:57 +0000950 CPUBreakpoint *bp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +0000951
Andreas Färberf0c3c502013-08-26 21:22:53 +0200952 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
Andreas Färberb3310ab2013-09-02 17:26:20 +0200953 if (bp->flags & mask) {
954 cpu_breakpoint_remove_by_ref(cpu, bp);
955 }
aliguoric0ce9982008-11-25 22:13:57 +0000956 }
bellard4c3a88a2003-07-26 12:06:08 +0000957}
958
bellardc33a3462003-07-29 20:50:33 +0000959/* enable or disable single step mode. EXCP_DEBUG is returned by the
960 CPU loop after each instruction */
Andreas Färber3825b282013-06-24 18:41:06 +0200961void cpu_single_step(CPUState *cpu, int enabled)
bellardc33a3462003-07-29 20:50:33 +0000962{
Andreas Färbered2803d2013-06-21 20:20:45 +0200963 if (cpu->singlestep_enabled != enabled) {
964 cpu->singlestep_enabled = enabled;
965 if (kvm_enabled()) {
Stefan Weil38e478e2013-07-25 20:50:21 +0200966 kvm_update_guest_debug(cpu, 0);
Andreas Färbered2803d2013-06-21 20:20:45 +0200967 } else {
Stuart Bradyccbb4d42009-05-03 12:15:06 +0100968 /* must flush all the translated code to avoid inconsistencies */
aliguorie22a25c2009-03-12 20:12:48 +0000969 /* XXX: only flush what is necessary */
Peter Crosthwaitebbd77c12015-06-23 19:31:15 -0700970 tb_flush(cpu);
aliguorie22a25c2009-03-12 20:12:48 +0000971 }
bellardc33a3462003-07-29 20:50:33 +0000972 }
bellardc33a3462003-07-29 20:50:33 +0000973}
974
Andreas Färbera47dddd2013-09-03 17:38:47 +0200975void cpu_abort(CPUState *cpu, const char *fmt, ...)
bellard75012672003-06-21 13:11:07 +0000976{
977 va_list ap;
pbrook493ae1f2007-11-23 16:53:59 +0000978 va_list ap2;
bellard75012672003-06-21 13:11:07 +0000979
980 va_start(ap, fmt);
pbrook493ae1f2007-11-23 16:53:59 +0000981 va_copy(ap2, ap);
bellard75012672003-06-21 13:11:07 +0000982 fprintf(stderr, "qemu: fatal: ");
983 vfprintf(stderr, fmt, ap);
984 fprintf(stderr, "\n");
Andreas Färber878096e2013-05-27 01:33:50 +0200985 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
Paolo Bonzini013a2942015-11-13 13:16:27 +0100986 if (qemu_log_separate()) {
Richard Henderson1ee73212016-09-22 15:17:10 -0700987 qemu_log_lock();
aliguori93fcfe32009-01-15 22:34:14 +0000988 qemu_log("qemu: fatal: ");
989 qemu_log_vprintf(fmt, ap2);
990 qemu_log("\n");
Andreas Färbera0762852013-06-16 07:28:50 +0200991 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
aliguori31b1a7b2009-01-15 22:35:09 +0000992 qemu_log_flush();
Richard Henderson1ee73212016-09-22 15:17:10 -0700993 qemu_log_unlock();
aliguori93fcfe32009-01-15 22:34:14 +0000994 qemu_log_close();
balrog924edca2007-06-10 14:07:13 +0000995 }
pbrook493ae1f2007-11-23 16:53:59 +0000996 va_end(ap2);
j_mayerf9373292007-09-29 12:18:20 +0000997 va_end(ap);
Pavel Dovgalyuk76159362015-09-17 19:25:07 +0300998 replay_finish();
Riku Voipiofd052bf2010-01-25 14:30:49 +0200999#if defined(CONFIG_USER_ONLY)
1000 {
1001 struct sigaction act;
1002 sigfillset(&act.sa_mask);
1003 act.sa_handler = SIG_DFL;
1004 sigaction(SIGABRT, &act, NULL);
1005 }
1006#endif
bellard75012672003-06-21 13:11:07 +00001007 abort();
1008}
1009
bellard01243112004-01-04 15:48:17 +00001010#if !defined(CONFIG_USER_ONLY)
Mike Day0dc3f442013-09-05 14:41:35 -04001011/* Called from RCU critical section */
Paolo Bonzini041603f2013-09-09 17:49:45 +02001012static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1013{
1014 RAMBlock *block;
1015
Paolo Bonzini43771532013-09-09 17:58:40 +02001016 block = atomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001017 if (block && addr - block->offset < block->max_length) {
Paolo Bonzini68851b92015-10-22 13:51:30 +02001018 return block;
Paolo Bonzini041603f2013-09-09 17:49:45 +02001019 }
Peter Xu99e15582017-05-12 12:17:39 +08001020 RAMBLOCK_FOREACH(block) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001021 if (addr - block->offset < block->max_length) {
Paolo Bonzini041603f2013-09-09 17:49:45 +02001022 goto found;
1023 }
1024 }
1025
1026 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1027 abort();
1028
1029found:
Paolo Bonzini43771532013-09-09 17:58:40 +02001030 /* It is safe to write mru_block outside the iothread lock. This
1031 * is what happens:
1032 *
1033 * mru_block = xxx
1034 * rcu_read_unlock()
1035 * xxx removed from list
1036 * rcu_read_lock()
1037 * read mru_block
1038 * mru_block = NULL;
1039 * call_rcu(reclaim_ramblock, xxx);
1040 * rcu_read_unlock()
1041 *
1042 * atomic_rcu_set is not needed here. The block was already published
1043 * when it was placed into the list. Here we're just making an extra
1044 * copy of the pointer.
1045 */
Paolo Bonzini041603f2013-09-09 17:49:45 +02001046 ram_list.mru_block = block;
1047 return block;
1048}
1049
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +02001050static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
bellard1ccde1c2004-02-06 19:46:14 +00001051{
Peter Crosthwaite9a135652015-09-10 22:39:41 -07001052 CPUState *cpu;
Paolo Bonzini041603f2013-09-09 17:49:45 +02001053 ram_addr_t start1;
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +02001054 RAMBlock *block;
1055 ram_addr_t end;
1056
1057 end = TARGET_PAGE_ALIGN(start + length);
1058 start &= TARGET_PAGE_MASK;
bellardf23db162005-08-21 19:12:28 +00001059
Mike Day0dc3f442013-09-05 14:41:35 -04001060 rcu_read_lock();
Paolo Bonzini041603f2013-09-09 17:49:45 +02001061 block = qemu_get_ram_block(start);
1062 assert(block == qemu_get_ram_block(end - 1));
Michael S. Tsirkin1240be22014-11-12 11:44:41 +02001063 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
Peter Crosthwaite9a135652015-09-10 22:39:41 -07001064 CPU_FOREACH(cpu) {
1065 tlb_reset_dirty(cpu, start1, length);
1066 }
Mike Day0dc3f442013-09-05 14:41:35 -04001067 rcu_read_unlock();
Juan Quintelad24981d2012-05-22 00:42:40 +02001068}
1069
1070/* Note: start and end must be within the same ram block. */
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +00001071bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
1072 ram_addr_t length,
1073 unsigned client)
Juan Quintelad24981d2012-05-22 00:42:40 +02001074{
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001075 DirtyMemoryBlocks *blocks;
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +00001076 unsigned long end, page;
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001077 bool dirty = false;
Juan Quintelad24981d2012-05-22 00:42:40 +02001078
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +00001079 if (length == 0) {
1080 return false;
1081 }
1082
1083 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
1084 page = start >> TARGET_PAGE_BITS;
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001085
1086 rcu_read_lock();
1087
1088 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1089
1090 while (page < end) {
1091 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1092 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1093 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1094
1095 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
1096 offset, num);
1097 page += num;
1098 }
1099
1100 rcu_read_unlock();
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +00001101
1102 if (dirty && tcg_enabled()) {
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +02001103 tlb_reset_dirty_range_all(start, length);
Juan Quintelad24981d2012-05-22 00:42:40 +02001104 }
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +00001105
1106 return dirty;
bellard1ccde1c2004-02-06 19:46:14 +00001107}
1108
Gerd Hoffmann8deaf122017-04-21 11:16:25 +02001109DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
1110 (ram_addr_t start, ram_addr_t length, unsigned client)
1111{
1112 DirtyMemoryBlocks *blocks;
1113 unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
1114 ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
1115 ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
1116 DirtyBitmapSnapshot *snap;
1117 unsigned long page, end, dest;
1118
1119 snap = g_malloc0(sizeof(*snap) +
1120 ((last - first) >> (TARGET_PAGE_BITS + 3)));
1121 snap->start = first;
1122 snap->end = last;
1123
1124 page = first >> TARGET_PAGE_BITS;
1125 end = last >> TARGET_PAGE_BITS;
1126 dest = 0;
1127
1128 rcu_read_lock();
1129
1130 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1131
1132 while (page < end) {
1133 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1134 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1135 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1136
1137 assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
1138 assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
1139 offset >>= BITS_PER_LEVEL;
1140
1141 bitmap_copy_and_clear_atomic(snap->dirty + dest,
1142 blocks->blocks[idx] + offset,
1143 num);
1144 page += num;
1145 dest += num >> BITS_PER_LEVEL;
1146 }
1147
1148 rcu_read_unlock();
1149
1150 if (tcg_enabled()) {
1151 tlb_reset_dirty_range_all(start, length);
1152 }
1153
1154 return snap;
1155}
1156
1157bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
1158 ram_addr_t start,
1159 ram_addr_t length)
1160{
1161 unsigned long page, end;
1162
1163 assert(start >= snap->start);
1164 assert(start + length <= snap->end);
1165
1166 end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
1167 page = (start - snap->start) >> TARGET_PAGE_BITS;
1168
1169 while (page < end) {
1170 if (test_bit(page, snap->dirty)) {
1171 return true;
1172 }
1173 page++;
1174 }
1175 return false;
1176}
1177
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01001178/* Called from RCU critical section */
Andreas Färberbb0e6272013-09-03 13:32:01 +02001179hwaddr memory_region_section_get_iotlb(CPUState *cpu,
Paolo Bonzini149f54b2013-05-24 12:59:37 +02001180 MemoryRegionSection *section,
1181 target_ulong vaddr,
1182 hwaddr paddr, hwaddr xlat,
1183 int prot,
1184 target_ulong *address)
Blue Swirle5548612012-04-21 13:08:33 +00001185{
Avi Kivitya8170e52012-10-23 12:30:10 +02001186 hwaddr iotlb;
Blue Swirle5548612012-04-21 13:08:33 +00001187 CPUWatchpoint *wp;
1188
Blue Swirlcc5bea62012-04-14 14:56:48 +00001189 if (memory_region_is_ram(section->mr)) {
Blue Swirle5548612012-04-21 13:08:33 +00001190 /* Normal RAM. */
Paolo Bonzinie4e69792016-03-01 10:44:50 +01001191 iotlb = memory_region_get_ram_addr(section->mr) + xlat;
Blue Swirle5548612012-04-21 13:08:33 +00001192 if (!section->readonly) {
Liu Ping Fanb41aac42013-05-29 11:09:17 +02001193 iotlb |= PHYS_SECTION_NOTDIRTY;
Blue Swirle5548612012-04-21 13:08:33 +00001194 } else {
Liu Ping Fanb41aac42013-05-29 11:09:17 +02001195 iotlb |= PHYS_SECTION_ROM;
Blue Swirle5548612012-04-21 13:08:33 +00001196 }
1197 } else {
Peter Maydell0b8e2c12015-07-20 12:27:16 +01001198 AddressSpaceDispatch *d;
1199
1200 d = atomic_rcu_read(&section->address_space->dispatch);
1201 iotlb = section - d->map.sections;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02001202 iotlb += xlat;
Blue Swirle5548612012-04-21 13:08:33 +00001203 }
1204
1205 /* Make accesses to pages with watchpoints go via the
1206 watchpoint trap routines. */
Andreas Färberff4700b2013-08-26 18:23:18 +02001207 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +01001208 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
Blue Swirle5548612012-04-21 13:08:33 +00001209 /* Avoid trapping reads of pages with a write breakpoint. */
1210 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
Liu Ping Fanb41aac42013-05-29 11:09:17 +02001211 iotlb = PHYS_SECTION_WATCH + paddr;
Blue Swirle5548612012-04-21 13:08:33 +00001212 *address |= TLB_MMIO;
1213 break;
1214 }
1215 }
1216 }
1217
1218 return iotlb;
1219}
bellard9fa3e852004-01-04 18:06:42 +00001220#endif /* defined(CONFIG_USER_ONLY) */
1221
pbrooke2eef172008-06-08 01:09:01 +00001222#if !defined(CONFIG_USER_ONLY)
pbrook8da3ff12008-12-01 18:59:50 +00001223
Anthony Liguoric227f092009-10-01 16:12:16 -05001224static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
Avi Kivity5312bd82012-02-12 18:32:55 +02001225 uint16_t section);
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001226static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
Avi Kivity54688b12012-02-09 17:34:32 +02001227
Igor Mammedova2b257d2014-10-31 16:38:37 +00001228static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1229 qemu_anon_ram_alloc;
Markus Armbruster91138032013-07-31 15:11:08 +02001230
1231/*
1232 * Set a custom physical guest memory alloator.
1233 * Accelerators with unusual needs may need this. Hopefully, we can
1234 * get rid of it eventually.
1235 */
Igor Mammedova2b257d2014-10-31 16:38:37 +00001236void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
Markus Armbruster91138032013-07-31 15:11:08 +02001237{
1238 phys_mem_alloc = alloc;
1239}
1240
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001241static uint16_t phys_section_add(PhysPageMap *map,
1242 MemoryRegionSection *section)
Avi Kivity5312bd82012-02-12 18:32:55 +02001243{
Paolo Bonzini68f3f652013-05-07 11:30:23 +02001244 /* The physical section number is ORed with a page-aligned
1245 * pointer to produce the iotlb entries. Thus it should
1246 * never overflow into the page-aligned value.
1247 */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001248 assert(map->sections_nb < TARGET_PAGE_SIZE);
Paolo Bonzini68f3f652013-05-07 11:30:23 +02001249
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001250 if (map->sections_nb == map->sections_nb_alloc) {
1251 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1252 map->sections = g_renew(MemoryRegionSection, map->sections,
1253 map->sections_nb_alloc);
Avi Kivity5312bd82012-02-12 18:32:55 +02001254 }
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001255 map->sections[map->sections_nb] = *section;
Paolo Bonzinidfde4e62013-05-06 10:46:11 +02001256 memory_region_ref(section->mr);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001257 return map->sections_nb++;
Avi Kivity5312bd82012-02-12 18:32:55 +02001258}
1259
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001260static void phys_section_destroy(MemoryRegion *mr)
1261{
Don Slutz55b4e802015-11-30 17:11:04 -05001262 bool have_sub_page = mr->subpage;
1263
Paolo Bonzinidfde4e62013-05-06 10:46:11 +02001264 memory_region_unref(mr);
1265
Don Slutz55b4e802015-11-30 17:11:04 -05001266 if (have_sub_page) {
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001267 subpage_t *subpage = container_of(mr, subpage_t, iomem);
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07001268 object_unref(OBJECT(&subpage->iomem));
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001269 g_free(subpage);
1270 }
1271}
1272
Paolo Bonzini60926662013-05-29 12:30:26 +02001273static void phys_sections_free(PhysPageMap *map)
Avi Kivity5312bd82012-02-12 18:32:55 +02001274{
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02001275 while (map->sections_nb > 0) {
1276 MemoryRegionSection *section = &map->sections[--map->sections_nb];
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001277 phys_section_destroy(section->mr);
1278 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02001279 g_free(map->sections);
1280 g_free(map->nodes);
Avi Kivity5312bd82012-02-12 18:32:55 +02001281}
1282
Avi Kivityac1970f2012-10-03 16:22:53 +02001283static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001284{
1285 subpage_t *subpage;
Avi Kivitya8170e52012-10-23 12:30:10 +02001286 hwaddr base = section->offset_within_address_space
Avi Kivity0f0cb162012-02-13 17:14:32 +02001287 & TARGET_PAGE_MASK;
Peter Xu003a0cf2017-05-15 16:50:57 +08001288 MemoryRegionSection *existing = phys_page_find(d, base);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001289 MemoryRegionSection subsection = {
1290 .offset_within_address_space = base,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001291 .size = int128_make64(TARGET_PAGE_SIZE),
Avi Kivity0f0cb162012-02-13 17:14:32 +02001292 };
Avi Kivitya8170e52012-10-23 12:30:10 +02001293 hwaddr start, end;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001294
Avi Kivityf3705d52012-03-08 16:16:34 +02001295 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001296
Avi Kivityf3705d52012-03-08 16:16:34 +02001297 if (!(existing->mr->subpage)) {
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001298 subpage = subpage_init(d->as, base);
Edgar E. Iglesias3be91e82013-11-07 18:42:51 +01001299 subsection.address_space = d->as;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001300 subsection.mr = &subpage->iomem;
Avi Kivityac1970f2012-10-03 16:22:53 +02001301 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001302 phys_section_add(&d->map, &subsection));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001303 } else {
Avi Kivityf3705d52012-03-08 16:16:34 +02001304 subpage = container_of(existing->mr, subpage_t, iomem);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001305 }
1306 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001307 end = start + int128_get64(section->size) - 1;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001308 subpage_register(subpage, start, end,
1309 phys_section_add(&d->map, section));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001310}
1311
1312
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001313static void register_multipage(AddressSpaceDispatch *d,
1314 MemoryRegionSection *section)
bellard33417e72003-08-10 21:47:01 +00001315{
Avi Kivitya8170e52012-10-23 12:30:10 +02001316 hwaddr start_addr = section->offset_within_address_space;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001317 uint16_t section_index = phys_section_add(&d->map, section);
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001318 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1319 TARGET_PAGE_BITS));
Avi Kivitydd811242012-01-02 12:17:03 +02001320
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001321 assert(num_pages);
1322 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
bellard33417e72003-08-10 21:47:01 +00001323}
1324
Avi Kivityac1970f2012-10-03 16:22:53 +02001325static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001326{
Paolo Bonzini89ae3372013-06-02 10:39:07 +02001327 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Paolo Bonzini00752702013-05-29 12:13:54 +02001328 AddressSpaceDispatch *d = as->next_dispatch;
Paolo Bonzini99b9cc02013-05-27 13:18:01 +02001329 MemoryRegionSection now = *section, remain = *section;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001330 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001331
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001332 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1333 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1334 - now.offset_within_address_space;
1335
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001336 now.size = int128_min(int128_make64(left), now.size);
Avi Kivityac1970f2012-10-03 16:22:53 +02001337 register_subpage(d, &now);
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001338 } else {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001339 now.size = int128_zero();
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001340 }
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001341 while (int128_ne(remain.size, now.size)) {
1342 remain.size = int128_sub(remain.size, now.size);
1343 remain.offset_within_address_space += int128_get64(now.size);
1344 remain.offset_within_region += int128_get64(now.size);
Tyler Hall69b67642012-07-25 18:45:04 -04001345 now = remain;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001346 if (int128_lt(remain.size, page_size)) {
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001347 register_subpage(d, &now);
Hu Tao88266242013-08-29 18:21:16 +08001348 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001349 now.size = page_size;
Avi Kivityac1970f2012-10-03 16:22:53 +02001350 register_subpage(d, &now);
Tyler Hall69b67642012-07-25 18:45:04 -04001351 } else {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001352 now.size = int128_and(now.size, int128_neg(page_size));
Avi Kivityac1970f2012-10-03 16:22:53 +02001353 register_multipage(d, &now);
Tyler Hall69b67642012-07-25 18:45:04 -04001354 }
Avi Kivity0f0cb162012-02-13 17:14:32 +02001355 }
1356}
1357
Sheng Yang62a27442010-01-26 19:21:16 +08001358void qemu_flush_coalesced_mmio_buffer(void)
1359{
1360 if (kvm_enabled())
1361 kvm_flush_coalesced_mmio_buffer();
1362}
1363
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001364void qemu_mutex_lock_ramlist(void)
1365{
1366 qemu_mutex_lock(&ram_list.mutex);
1367}
1368
1369void qemu_mutex_unlock_ramlist(void)
1370{
1371 qemu_mutex_unlock(&ram_list.mutex);
1372}
1373
Peter Xube9b23c2017-05-12 12:17:41 +08001374void ram_block_dump(Monitor *mon)
1375{
1376 RAMBlock *block;
1377 char *psize;
1378
1379 rcu_read_lock();
1380 monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
1381 "Block Name", "PSize", "Offset", "Used", "Total");
1382 RAMBLOCK_FOREACH(block) {
1383 psize = size_to_str(block->page_size);
1384 monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
1385 " 0x%016" PRIx64 "\n", block->idstr, psize,
1386 (uint64_t)block->offset,
1387 (uint64_t)block->used_length,
1388 (uint64_t)block->max_length);
1389 g_free(psize);
1390 }
1391 rcu_read_unlock();
1392}
1393
Markus Armbrustere1e84ba2013-07-31 15:11:10 +02001394#ifdef __linux__
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001395/*
1396 * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
1397 * may or may not name the same files / on the same filesystem now as
1398 * when we actually open and map them. Iterate over the file
1399 * descriptors instead, and use qemu_fd_getpagesize().
1400 */
1401static int find_max_supported_pagesize(Object *obj, void *opaque)
1402{
1403 char *mem_path;
1404 long *hpsize_min = opaque;
1405
1406 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1407 mem_path = object_property_get_str(obj, "mem-path", NULL);
1408 if (mem_path) {
1409 long hpsize = qemu_mempath_getpagesize(mem_path);
1410 if (hpsize < *hpsize_min) {
1411 *hpsize_min = hpsize;
1412 }
1413 } else {
1414 *hpsize_min = getpagesize();
1415 }
1416 }
1417
1418 return 0;
1419}
1420
1421long qemu_getrampagesize(void)
1422{
1423 long hpsize = LONG_MAX;
1424 long mainrampagesize;
1425 Object *memdev_root;
1426
1427 if (mem_path) {
1428 mainrampagesize = qemu_mempath_getpagesize(mem_path);
1429 } else {
1430 mainrampagesize = getpagesize();
1431 }
1432
1433 /* it's possible we have memory-backend objects with
1434 * hugepage-backed RAM. these may get mapped into system
1435 * address space via -numa parameters or memory hotplug
1436 * hooks. we want to take these into account, but we
1437 * also want to make sure these supported hugepage
1438 * sizes are applicable across the entire range of memory
1439 * we may boot from, so we take the min across all
1440 * backends, and assume normal pages in cases where a
1441 * backend isn't backed by hugepages.
1442 */
1443 memdev_root = object_resolve_path("/objects", NULL);
1444 if (memdev_root) {
1445 object_child_foreach(memdev_root, find_max_supported_pagesize, &hpsize);
1446 }
1447 if (hpsize == LONG_MAX) {
1448 /* No additional memory regions found ==> Report main RAM page size */
1449 return mainrampagesize;
1450 }
1451
1452 /* If NUMA is disabled or the NUMA nodes are not backed with a
1453 * memory-backend, then there is at least one node using "normal" RAM,
1454 * so if its page size is smaller we have got to report that size instead.
1455 */
1456 if (hpsize > mainrampagesize &&
1457 (nb_numa_nodes == 0 || numa_info[0].node_memdev == NULL)) {
1458 static bool warned;
1459 if (!warned) {
1460 error_report("Huge page support disabled (n/a for main memory).");
1461 warned = true;
1462 }
1463 return mainrampagesize;
1464 }
1465
1466 return hpsize;
1467}
1468#else
1469long qemu_getrampagesize(void)
1470{
1471 return getpagesize();
1472}
1473#endif
1474
1475#ifdef __linux__
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001476static int64_t get_file_size(int fd)
1477{
1478 int64_t size = lseek(fd, 0, SEEK_END);
1479 if (size < 0) {
1480 return -errno;
1481 }
1482 return size;
1483}
1484
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001485static int file_ram_open(const char *path,
1486 const char *region_name,
1487 bool *created,
1488 Error **errp)
Marcelo Tosattic9027602010-03-01 20:25:08 -03001489{
1490 char *filename;
Peter Feiner8ca761f2013-03-04 13:54:25 -05001491 char *sanitized_name;
1492 char *c;
Paolo Bonzini5c3ece72016-03-17 15:53:13 +01001493 int fd = -1;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001494
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001495 *created = false;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001496 for (;;) {
1497 fd = open(path, O_RDWR);
1498 if (fd >= 0) {
1499 /* @path names an existing file, use it */
1500 break;
1501 }
1502 if (errno == ENOENT) {
1503 /* @path names a file that doesn't exist, create it */
1504 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1505 if (fd >= 0) {
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001506 *created = true;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001507 break;
1508 }
1509 } else if (errno == EISDIR) {
1510 /* @path names a directory, create a file there */
1511 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001512 sanitized_name = g_strdup(region_name);
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001513 for (c = sanitized_name; *c != '\0'; c++) {
1514 if (*c == '/') {
1515 *c = '_';
1516 }
1517 }
1518
1519 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1520 sanitized_name);
1521 g_free(sanitized_name);
1522
1523 fd = mkstemp(filename);
1524 if (fd >= 0) {
1525 unlink(filename);
1526 g_free(filename);
1527 break;
1528 }
1529 g_free(filename);
1530 }
1531 if (errno != EEXIST && errno != EINTR) {
1532 error_setg_errno(errp, errno,
1533 "can't open backing store %s for guest RAM",
1534 path);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001535 return -1;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001536 }
1537 /*
1538 * Try again on EINTR and EEXIST. The latter happens when
1539 * something else creates the file between our two open().
1540 */
1541 }
1542
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001543 return fd;
1544}
1545
1546static void *file_ram_alloc(RAMBlock *block,
1547 ram_addr_t memory,
1548 int fd,
1549 bool truncate,
1550 Error **errp)
1551{
1552 void *area;
1553
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001554 block->page_size = qemu_fd_getpagesize(fd);
Haozhong Zhang83606682016-10-24 20:49:37 +08001555 block->mr->align = block->page_size;
1556#if defined(__s390x__)
1557 if (kvm_enabled()) {
1558 block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1559 }
1560#endif
Marcelo Tosattic9027602010-03-01 20:25:08 -03001561
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001562 if (memory < block->page_size) {
Hu Tao557529d2014-09-09 13:28:00 +08001563 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001564 "or larger than page size 0x%zx",
1565 memory, block->page_size);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001566 return NULL;
Haozhong Zhang1775f112016-11-02 09:05:51 +08001567 }
1568
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001569 memory = ROUND_UP(memory, block->page_size);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001570
1571 /*
1572 * ftruncate is not supported by hugetlbfs in older
1573 * hosts, so don't bother bailing out on errors.
1574 * If anything goes wrong with it under other filesystems,
1575 * mmap will fail.
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001576 *
1577 * Do not truncate the non-empty backend file to avoid corrupting
1578 * the existing data in the file. Disabling shrinking is not
1579 * enough. For example, the current vNVDIMM implementation stores
1580 * the guest NVDIMM labels at the end of the backend file. If the
1581 * backend file is later extended, QEMU will not be able to find
1582 * those labels. Therefore, extending the non-empty backend file
1583 * is disabled as well.
Marcelo Tosattic9027602010-03-01 20:25:08 -03001584 */
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001585 if (truncate && ftruncate(fd, memory)) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001586 perror("ftruncate");
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001587 }
Marcelo Tosattic9027602010-03-01 20:25:08 -03001588
Dominik Dingeld2f39ad2016-04-25 13:55:38 +02001589 area = qemu_ram_mmap(fd, memory, block->mr->align,
1590 block->flags & RAM_SHARED);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001591 if (area == MAP_FAILED) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001592 error_setg_errno(errp, errno,
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001593 "unable to map backing store for guest RAM");
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001594 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001595 }
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001596
1597 if (mem_prealloc) {
Jitendra Kolhe1e356fc2017-02-24 09:01:43 +05301598 os_mem_prealloc(fd, area, memory, smp_cpus, errp);
Igor Mammedov056b68a2016-07-20 11:54:03 +02001599 if (errp && *errp) {
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001600 qemu_ram_munmap(area, memory);
1601 return NULL;
Igor Mammedov056b68a2016-07-20 11:54:03 +02001602 }
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001603 }
1604
Alex Williamson04b16652010-07-02 11:13:17 -06001605 block->fd = fd;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001606 return area;
1607}
1608#endif
1609
Mike Day0dc3f442013-09-05 14:41:35 -04001610/* Called with the ramlist lock held. */
Alex Williamsond17b5282010-06-25 11:08:38 -06001611static ram_addr_t find_ram_offset(ram_addr_t size)
1612{
Alex Williamson04b16652010-07-02 11:13:17 -06001613 RAMBlock *block, *next_block;
Alex Williamson3e837b22011-10-31 08:54:09 -06001614 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001615
Stefan Hajnoczi49cd9ac2013-03-11 10:20:21 +01001616 assert(size != 0); /* it would hand out same offset multiple times */
1617
Mike Day0dc3f442013-09-05 14:41:35 -04001618 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
Alex Williamson04b16652010-07-02 11:13:17 -06001619 return 0;
Mike Day0d53d9f2015-01-21 13:45:24 +01001620 }
Alex Williamson04b16652010-07-02 11:13:17 -06001621
Peter Xu99e15582017-05-12 12:17:39 +08001622 RAMBLOCK_FOREACH(block) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00001623 ram_addr_t end, next = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001624
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001625 end = block->offset + block->max_length;
Alex Williamson04b16652010-07-02 11:13:17 -06001626
Peter Xu99e15582017-05-12 12:17:39 +08001627 RAMBLOCK_FOREACH(next_block) {
Alex Williamson04b16652010-07-02 11:13:17 -06001628 if (next_block->offset >= end) {
1629 next = MIN(next, next_block->offset);
1630 }
1631 }
1632 if (next - end >= size && next - end < mingap) {
Alex Williamson3e837b22011-10-31 08:54:09 -06001633 offset = end;
Alex Williamson04b16652010-07-02 11:13:17 -06001634 mingap = next - end;
1635 }
1636 }
Alex Williamson3e837b22011-10-31 08:54:09 -06001637
1638 if (offset == RAM_ADDR_MAX) {
1639 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1640 (uint64_t)size);
1641 abort();
1642 }
1643
Alex Williamson04b16652010-07-02 11:13:17 -06001644 return offset;
1645}
1646
Juan Quintelab8c48992017-03-21 17:44:30 +01001647unsigned long last_ram_page(void)
Alex Williamson04b16652010-07-02 11:13:17 -06001648{
Alex Williamsond17b5282010-06-25 11:08:38 -06001649 RAMBlock *block;
1650 ram_addr_t last = 0;
1651
Mike Day0dc3f442013-09-05 14:41:35 -04001652 rcu_read_lock();
Peter Xu99e15582017-05-12 12:17:39 +08001653 RAMBLOCK_FOREACH(block) {
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001654 last = MAX(last, block->offset + block->max_length);
Mike Day0d53d9f2015-01-21 13:45:24 +01001655 }
Mike Day0dc3f442013-09-05 14:41:35 -04001656 rcu_read_unlock();
Juan Quintelab8c48992017-03-21 17:44:30 +01001657 return last >> TARGET_PAGE_BITS;
Alex Williamsond17b5282010-06-25 11:08:38 -06001658}
1659
Jason Baronddb97f12012-08-02 15:44:16 -04001660static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1661{
1662 int ret;
Jason Baronddb97f12012-08-02 15:44:16 -04001663
1664 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +02001665 if (!machine_dump_guest_core(current_machine)) {
Jason Baronddb97f12012-08-02 15:44:16 -04001666 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1667 if (ret) {
1668 perror("qemu_madvise");
1669 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1670 "but dump_guest_core=off specified\n");
1671 }
1672 }
1673}
1674
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00001675const char *qemu_ram_get_idstr(RAMBlock *rb)
1676{
1677 return rb->idstr;
1678}
1679
Dr. David Alan Gilbert463a4ac2017-03-07 18:36:36 +00001680bool qemu_ram_is_shared(RAMBlock *rb)
1681{
1682 return rb->flags & RAM_SHARED;
1683}
1684
Mike Dayae3a7042013-09-05 14:41:35 -04001685/* Called with iothread lock held. */
Gongleifa53a0e2016-05-10 10:04:59 +08001686void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
Hu Tao20cfe882014-04-02 15:13:26 +08001687{
Gongleifa53a0e2016-05-10 10:04:59 +08001688 RAMBlock *block;
Hu Tao20cfe882014-04-02 15:13:26 +08001689
Avi Kivityc5705a72011-12-20 15:59:12 +02001690 assert(new_block);
1691 assert(!new_block->idstr[0]);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001692
Anthony Liguori09e5ab62012-02-03 12:28:43 -06001693 if (dev) {
1694 char *id = qdev_get_dev_path(dev);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001695 if (id) {
1696 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
Anthony Liguori7267c092011-08-20 22:09:37 -05001697 g_free(id);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001698 }
1699 }
1700 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1701
Gongleiab0a9952016-05-10 10:05:00 +08001702 rcu_read_lock();
Peter Xu99e15582017-05-12 12:17:39 +08001703 RAMBLOCK_FOREACH(block) {
Gongleifa53a0e2016-05-10 10:04:59 +08001704 if (block != new_block &&
1705 !strcmp(block->idstr, new_block->idstr)) {
Cam Macdonell84b89d72010-07-26 18:10:57 -06001706 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1707 new_block->idstr);
1708 abort();
1709 }
1710 }
Mike Day0dc3f442013-09-05 14:41:35 -04001711 rcu_read_unlock();
Avi Kivityc5705a72011-12-20 15:59:12 +02001712}
1713
Mike Dayae3a7042013-09-05 14:41:35 -04001714/* Called with iothread lock held. */
Gongleifa53a0e2016-05-10 10:04:59 +08001715void qemu_ram_unset_idstr(RAMBlock *block)
Hu Tao20cfe882014-04-02 15:13:26 +08001716{
Mike Dayae3a7042013-09-05 14:41:35 -04001717 /* FIXME: arch_init.c assumes that this is not called throughout
1718 * migration. Ignore the problem since hot-unplug during migration
1719 * does not work anyway.
1720 */
Hu Tao20cfe882014-04-02 15:13:26 +08001721 if (block) {
1722 memset(block->idstr, 0, sizeof(block->idstr));
1723 }
1724}
1725
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001726size_t qemu_ram_pagesize(RAMBlock *rb)
1727{
1728 return rb->page_size;
1729}
1730
Dr. David Alan Gilbert67f11b52017-02-24 18:28:34 +00001731/* Returns the largest size of page in use */
1732size_t qemu_ram_pagesize_largest(void)
1733{
1734 RAMBlock *block;
1735 size_t largest = 0;
1736
Peter Xu99e15582017-05-12 12:17:39 +08001737 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilbert67f11b52017-02-24 18:28:34 +00001738 largest = MAX(largest, qemu_ram_pagesize(block));
1739 }
1740
1741 return largest;
1742}
1743
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001744static int memory_try_enable_merging(void *addr, size_t len)
1745{
Marcel Apfelbaum75cc7f02015-02-04 17:43:55 +02001746 if (!machine_mem_merge(current_machine)) {
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001747 /* disabled by the user */
1748 return 0;
1749 }
1750
1751 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1752}
1753
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001754/* Only legal before guest might have detected the memory size: e.g. on
1755 * incoming migration, or right after reset.
1756 *
1757 * As memory core doesn't know how is memory accessed, it is up to
1758 * resize callback to update device state and/or add assertions to detect
1759 * misuse, if necessary.
1760 */
Gongleifa53a0e2016-05-10 10:04:59 +08001761int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001762{
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001763 assert(block);
1764
Dr. David Alan Gilbert4ed023c2015-11-05 18:11:16 +00001765 newsize = HOST_PAGE_ALIGN(newsize);
Michael S. Tsirkin129ddaf2015-02-17 10:15:30 +01001766
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001767 if (block->used_length == newsize) {
1768 return 0;
1769 }
1770
1771 if (!(block->flags & RAM_RESIZEABLE)) {
1772 error_setg_errno(errp, EINVAL,
1773 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1774 " in != 0x" RAM_ADDR_FMT, block->idstr,
1775 newsize, block->used_length);
1776 return -EINVAL;
1777 }
1778
1779 if (block->max_length < newsize) {
1780 error_setg_errno(errp, EINVAL,
1781 "Length too large: %s: 0x" RAM_ADDR_FMT
1782 " > 0x" RAM_ADDR_FMT, block->idstr,
1783 newsize, block->max_length);
1784 return -EINVAL;
1785 }
1786
1787 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1788 block->used_length = newsize;
Paolo Bonzini58d27072015-03-23 11:56:01 +01001789 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1790 DIRTY_CLIENTS_ALL);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001791 memory_region_set_size(block->mr, newsize);
1792 if (block->resized) {
1793 block->resized(block->idstr, newsize, block->host);
1794 }
1795 return 0;
1796}
1797
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001798/* Called with ram_list.mutex held */
1799static void dirty_memory_extend(ram_addr_t old_ram_size,
1800 ram_addr_t new_ram_size)
1801{
1802 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
1803 DIRTY_MEMORY_BLOCK_SIZE);
1804 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
1805 DIRTY_MEMORY_BLOCK_SIZE);
1806 int i;
1807
1808 /* Only need to extend if block count increased */
1809 if (new_num_blocks <= old_num_blocks) {
1810 return;
1811 }
1812
1813 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1814 DirtyMemoryBlocks *old_blocks;
1815 DirtyMemoryBlocks *new_blocks;
1816 int j;
1817
1818 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
1819 new_blocks = g_malloc(sizeof(*new_blocks) +
1820 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1821
1822 if (old_num_blocks) {
1823 memcpy(new_blocks->blocks, old_blocks->blocks,
1824 old_num_blocks * sizeof(old_blocks->blocks[0]));
1825 }
1826
1827 for (j = old_num_blocks; j < new_num_blocks; j++) {
1828 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1829 }
1830
1831 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1832
1833 if (old_blocks) {
1834 g_free_rcu(old_blocks, rcu);
1835 }
1836 }
1837}
1838
Fam Zheng528f46a2016-03-01 14:18:18 +08001839static void ram_block_add(RAMBlock *new_block, Error **errp)
Avi Kivityc5705a72011-12-20 15:59:12 +02001840{
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001841 RAMBlock *block;
Mike Day0d53d9f2015-01-21 13:45:24 +01001842 RAMBlock *last_block = NULL;
Juan Quintela2152f5c2013-10-08 13:52:02 +02001843 ram_addr_t old_ram_size, new_ram_size;
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001844 Error *err = NULL;
Juan Quintela2152f5c2013-10-08 13:52:02 +02001845
Juan Quintelab8c48992017-03-21 17:44:30 +01001846 old_ram_size = last_ram_page();
Avi Kivityc5705a72011-12-20 15:59:12 +02001847
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001848 qemu_mutex_lock_ramlist();
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001849 new_block->offset = find_ram_offset(new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001850
1851 if (!new_block->host) {
1852 if (xen_enabled()) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001853 xen_ram_alloc(new_block->offset, new_block->max_length,
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001854 new_block->mr, &err);
1855 if (err) {
1856 error_propagate(errp, err);
1857 qemu_mutex_unlock_ramlist();
Paolo Bonzini39c350e2016-03-09 18:14:01 +01001858 return;
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001859 }
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001860 } else {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001861 new_block->host = phys_mem_alloc(new_block->max_length,
Igor Mammedova2b257d2014-10-31 16:38:37 +00001862 &new_block->mr->align);
Markus Armbruster39228252013-07-31 15:11:11 +02001863 if (!new_block->host) {
Hu Taoef701d72014-09-09 13:27:54 +08001864 error_setg_errno(errp, errno,
1865 "cannot set up guest memory '%s'",
1866 memory_region_name(new_block->mr));
1867 qemu_mutex_unlock_ramlist();
Paolo Bonzini39c350e2016-03-09 18:14:01 +01001868 return;
Markus Armbruster39228252013-07-31 15:11:11 +02001869 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001870 memory_try_enable_merging(new_block->host, new_block->max_length);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09001871 }
1872 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001873
Li Zhijiandd631692015-07-02 20:18:06 +08001874 new_ram_size = MAX(old_ram_size,
1875 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1876 if (new_ram_size > old_ram_size) {
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001877 dirty_memory_extend(old_ram_size, new_ram_size);
Li Zhijiandd631692015-07-02 20:18:06 +08001878 }
Mike Day0d53d9f2015-01-21 13:45:24 +01001879 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1880 * QLIST (which has an RCU-friendly variant) does not have insertion at
1881 * tail, so save the last element in last_block.
1882 */
Peter Xu99e15582017-05-12 12:17:39 +08001883 RAMBLOCK_FOREACH(block) {
Mike Day0d53d9f2015-01-21 13:45:24 +01001884 last_block = block;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001885 if (block->max_length < new_block->max_length) {
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001886 break;
1887 }
1888 }
1889 if (block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001890 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001891 } else if (last_block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001892 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001893 } else { /* list is empty */
Mike Day0dc3f442013-09-05 14:41:35 -04001894 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001895 }
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001896 ram_list.mru_block = NULL;
Cam Macdonell84b89d72010-07-26 18:10:57 -06001897
Mike Day0dc3f442013-09-05 14:41:35 -04001898 /* Write list before version */
1899 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001900 ram_list.version++;
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001901 qemu_mutex_unlock_ramlist();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001902
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001903 cpu_physical_memory_set_dirty_range(new_block->offset,
Paolo Bonzini58d27072015-03-23 11:56:01 +01001904 new_block->used_length,
1905 DIRTY_CLIENTS_ALL);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001906
Paolo Bonzinia904c912015-01-21 16:18:35 +01001907 if (new_block->host) {
1908 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1909 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
Cao jinc2cd6272016-09-12 14:34:56 +08001910 /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
Paolo Bonzinia904c912015-01-21 16:18:35 +01001911 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
Paolo Bonzini0987d732016-12-21 00:31:36 +08001912 ram_block_notify_add(new_block->host, new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001913 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001914}
1915
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08001916#ifdef __linux__
Fam Zheng528f46a2016-03-01 14:18:18 +08001917RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1918 bool share, const char *mem_path,
1919 Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001920{
1921 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001922 Error *local_err = NULL;
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001923 int fd;
1924 bool created;
1925 int64_t file_size;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001926
1927 if (xen_enabled()) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001928 error_setg(errp, "-mem-path not supported with Xen");
Fam Zheng528f46a2016-03-01 14:18:18 +08001929 return NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001930 }
1931
Marc-André Lureaue45e7ae2017-06-02 18:12:21 +04001932 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1933 error_setg(errp,
1934 "host lacks kvm mmu notifiers, -mem-path unsupported");
1935 return NULL;
1936 }
1937
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001938 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1939 /*
1940 * file_ram_alloc() needs to allocate just like
1941 * phys_mem_alloc, but we haven't bothered to provide
1942 * a hook there.
1943 */
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001944 error_setg(errp,
1945 "-mem-path not supported with this accelerator");
Fam Zheng528f46a2016-03-01 14:18:18 +08001946 return NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001947 }
1948
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001949 fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
1950 if (fd < 0) {
1951 return NULL;
1952 }
1953
Dr. David Alan Gilbert4ed023c2015-11-05 18:11:16 +00001954 size = HOST_PAGE_ALIGN(size);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001955 file_size = get_file_size(fd);
1956 if (file_size > 0 && file_size < size) {
1957 error_setg(errp, "backing store %s size 0x%" PRIx64
1958 " does not match 'size' option 0x" RAM_ADDR_FMT,
1959 mem_path, file_size, size);
1960 close(fd);
1961 return NULL;
1962 }
1963
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001964 new_block = g_malloc0(sizeof(*new_block));
1965 new_block->mr = mr;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001966 new_block->used_length = size;
1967 new_block->max_length = size;
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001968 new_block->flags = share ? RAM_SHARED : 0;
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001969 new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001970 if (!new_block->host) {
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001971 if (created) {
1972 unlink(mem_path);
1973 }
1974 close(fd);
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001975 g_free(new_block);
Fam Zheng528f46a2016-03-01 14:18:18 +08001976 return NULL;
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001977 }
1978
Fam Zheng528f46a2016-03-01 14:18:18 +08001979 ram_block_add(new_block, &local_err);
Hu Taoef701d72014-09-09 13:27:54 +08001980 if (local_err) {
1981 g_free(new_block);
1982 error_propagate(errp, local_err);
Fam Zheng528f46a2016-03-01 14:18:18 +08001983 return NULL;
Hu Taoef701d72014-09-09 13:27:54 +08001984 }
Fam Zheng528f46a2016-03-01 14:18:18 +08001985 return new_block;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001986}
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08001987#endif
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001988
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001989static
Fam Zheng528f46a2016-03-01 14:18:18 +08001990RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1991 void (*resized)(const char*,
1992 uint64_t length,
1993 void *host),
1994 void *host, bool resizeable,
1995 MemoryRegion *mr, Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001996{
1997 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001998 Error *local_err = NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001999
Dr. David Alan Gilbert4ed023c2015-11-05 18:11:16 +00002000 size = HOST_PAGE_ALIGN(size);
2001 max_size = HOST_PAGE_ALIGN(max_size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002002 new_block = g_malloc0(sizeof(*new_block));
2003 new_block->mr = mr;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002004 new_block->resized = resized;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002005 new_block->used_length = size;
2006 new_block->max_length = max_size;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002007 assert(max_size >= size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002008 new_block->fd = -1;
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01002009 new_block->page_size = getpagesize();
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002010 new_block->host = host;
2011 if (host) {
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08002012 new_block->flags |= RAM_PREALLOC;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002013 }
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002014 if (resizeable) {
2015 new_block->flags |= RAM_RESIZEABLE;
2016 }
Fam Zheng528f46a2016-03-01 14:18:18 +08002017 ram_block_add(new_block, &local_err);
Hu Taoef701d72014-09-09 13:27:54 +08002018 if (local_err) {
2019 g_free(new_block);
2020 error_propagate(errp, local_err);
Fam Zheng528f46a2016-03-01 14:18:18 +08002021 return NULL;
Hu Taoef701d72014-09-09 13:27:54 +08002022 }
Fam Zheng528f46a2016-03-01 14:18:18 +08002023 return new_block;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002024}
2025
Fam Zheng528f46a2016-03-01 14:18:18 +08002026RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002027 MemoryRegion *mr, Error **errp)
2028{
2029 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
2030}
2031
Fam Zheng528f46a2016-03-01 14:18:18 +08002032RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
pbrook94a6b542009-04-11 17:15:54 +00002033{
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002034 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
2035}
2036
Fam Zheng528f46a2016-03-01 14:18:18 +08002037RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002038 void (*resized)(const char*,
2039 uint64_t length,
2040 void *host),
2041 MemoryRegion *mr, Error **errp)
2042{
2043 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
pbrook94a6b542009-04-11 17:15:54 +00002044}
bellarde9a1ab12007-02-08 23:08:38 +00002045
Paolo Bonzini43771532013-09-09 17:58:40 +02002046static void reclaim_ramblock(RAMBlock *block)
2047{
2048 if (block->flags & RAM_PREALLOC) {
2049 ;
2050 } else if (xen_enabled()) {
2051 xen_invalidate_map_cache_entry(block->host);
2052#ifndef _WIN32
2053 } else if (block->fd >= 0) {
Eduardo Habkost2f3a2bb2015-11-06 20:11:21 -02002054 qemu_ram_munmap(block->host, block->max_length);
Paolo Bonzini43771532013-09-09 17:58:40 +02002055 close(block->fd);
2056#endif
2057 } else {
2058 qemu_anon_ram_free(block->host, block->max_length);
2059 }
2060 g_free(block);
2061}
2062
Fam Zhengf1060c52016-03-01 14:18:22 +08002063void qemu_ram_free(RAMBlock *block)
bellarde9a1ab12007-02-08 23:08:38 +00002064{
Marc-André Lureau85bc2a12016-03-29 13:20:51 +02002065 if (!block) {
2066 return;
2067 }
2068
Paolo Bonzini0987d732016-12-21 00:31:36 +08002069 if (block->host) {
2070 ram_block_notify_remove(block->host, block->max_length);
2071 }
2072
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07002073 qemu_mutex_lock_ramlist();
Fam Zhengf1060c52016-03-01 14:18:22 +08002074 QLIST_REMOVE_RCU(block, next);
2075 ram_list.mru_block = NULL;
2076 /* Write list before version */
2077 smp_wmb();
2078 ram_list.version++;
2079 call_rcu(block, reclaim_ramblock, rcu);
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07002080 qemu_mutex_unlock_ramlist();
bellarde9a1ab12007-02-08 23:08:38 +00002081}
2082
Huang Yingcd19cfa2011-03-02 08:56:19 +01002083#ifndef _WIN32
2084void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2085{
2086 RAMBlock *block;
2087 ram_addr_t offset;
2088 int flags;
2089 void *area, *vaddr;
2090
Peter Xu99e15582017-05-12 12:17:39 +08002091 RAMBLOCK_FOREACH(block) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01002092 offset = addr - block->offset;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002093 if (offset < block->max_length) {
Michael S. Tsirkin1240be22014-11-12 11:44:41 +02002094 vaddr = ramblock_ptr(block, offset);
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08002095 if (block->flags & RAM_PREALLOC) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01002096 ;
Markus Armbrusterdfeaf2a2013-07-31 15:11:05 +02002097 } else if (xen_enabled()) {
2098 abort();
Huang Yingcd19cfa2011-03-02 08:56:19 +01002099 } else {
2100 flags = MAP_FIXED;
Markus Armbruster3435f392013-07-31 15:11:07 +02002101 if (block->fd >= 0) {
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08002102 flags |= (block->flags & RAM_SHARED ?
2103 MAP_SHARED : MAP_PRIVATE);
Markus Armbruster3435f392013-07-31 15:11:07 +02002104 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2105 flags, block->fd, offset);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002106 } else {
Markus Armbruster2eb9fba2013-07-31 15:11:09 +02002107 /*
2108 * Remap needs to match alloc. Accelerators that
2109 * set phys_mem_alloc never remap. If they did,
2110 * we'd need a remap hook here.
2111 */
2112 assert(phys_mem_alloc == qemu_anon_ram_alloc);
2113
Huang Yingcd19cfa2011-03-02 08:56:19 +01002114 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
2115 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2116 flags, -1, 0);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002117 }
2118 if (area != vaddr) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00002119 fprintf(stderr, "Could not remap addr: "
2120 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
Huang Yingcd19cfa2011-03-02 08:56:19 +01002121 length, addr);
2122 exit(1);
2123 }
Luiz Capitulino8490fc72012-09-05 16:50:16 -03002124 memory_try_enable_merging(vaddr, length);
Jason Baronddb97f12012-08-02 15:44:16 -04002125 qemu_ram_setup_dump(vaddr, length);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002126 }
Huang Yingcd19cfa2011-03-02 08:56:19 +01002127 }
2128 }
2129}
2130#endif /* !_WIN32 */
2131
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002132/* Return a host pointer to ram allocated with qemu_ram_alloc.
Mike Dayae3a7042013-09-05 14:41:35 -04002133 * This should not be used for general purpose DMA. Use address_space_map
2134 * or address_space_rw instead. For local memory (e.g. video ram) that the
2135 * device owns, use memory_region_get_ram_ptr.
Mike Day0dc3f442013-09-05 14:41:35 -04002136 *
Paolo Bonzini49b24af2015-12-16 10:30:47 +01002137 * Called within RCU critical section.
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002138 */
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002139void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002140{
Gonglei3655cb92016-02-20 10:35:20 +08002141 RAMBlock *block = ram_block;
2142
2143 if (block == NULL) {
2144 block = qemu_get_ram_block(addr);
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002145 addr -= block->offset;
Gonglei3655cb92016-02-20 10:35:20 +08002146 }
Mike Dayae3a7042013-09-05 14:41:35 -04002147
2148 if (xen_enabled() && block->host == NULL) {
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01002149 /* We need to check if the requested address is in the RAM
2150 * because we don't want to map the entire memory in QEMU.
2151 * In that case just map until the end of the page.
2152 */
2153 if (block->offset == 0) {
Stefano Stabellini1ff7c592017-05-03 14:00:35 -07002154 return xen_map_cache(addr, 0, 0, false);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01002155 }
Mike Dayae3a7042013-09-05 14:41:35 -04002156
Stefano Stabellini1ff7c592017-05-03 14:00:35 -07002157 block->host = xen_map_cache(block->offset, block->max_length, 1, false);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01002158 }
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002159 return ramblock_ptr(block, addr);
pbrookdc828ca2009-04-09 22:21:07 +00002160}
2161
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002162/* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
Mike Dayae3a7042013-09-05 14:41:35 -04002163 * but takes a size argument.
Mike Day0dc3f442013-09-05 14:41:35 -04002164 *
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002165 * Called within RCU critical section.
Mike Dayae3a7042013-09-05 14:41:35 -04002166 */
Gonglei3655cb92016-02-20 10:35:20 +08002167static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
2168 hwaddr *size)
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002169{
Gonglei3655cb92016-02-20 10:35:20 +08002170 RAMBlock *block = ram_block;
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01002171 if (*size == 0) {
2172 return NULL;
2173 }
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002174
Gonglei3655cb92016-02-20 10:35:20 +08002175 if (block == NULL) {
2176 block = qemu_get_ram_block(addr);
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002177 addr -= block->offset;
Gonglei3655cb92016-02-20 10:35:20 +08002178 }
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002179 *size = MIN(*size, block->max_length - addr);
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002180
2181 if (xen_enabled() && block->host == NULL) {
2182 /* We need to check if the requested address is in the RAM
2183 * because we don't want to map the entire memory in QEMU.
2184 * In that case just map the requested area.
2185 */
2186 if (block->offset == 0) {
Stefano Stabellini1ff7c592017-05-03 14:00:35 -07002187 return xen_map_cache(addr, *size, 1, true);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002188 }
2189
Stefano Stabellini1ff7c592017-05-03 14:00:35 -07002190 block->host = xen_map_cache(block->offset, block->max_length, 1, true);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002191 }
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002192
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002193 return ramblock_ptr(block, addr);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002194}
2195
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002196/*
2197 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
2198 * in that RAMBlock.
2199 *
2200 * ptr: Host pointer to look up
2201 * round_offset: If true round the result offset down to a page boundary
2202 * *ram_addr: set to result ram_addr
2203 * *offset: set to result offset within the RAMBlock
2204 *
2205 * Returns: RAMBlock (or NULL if not found)
Mike Dayae3a7042013-09-05 14:41:35 -04002206 *
2207 * By the time this function returns, the returned pointer is not protected
2208 * by RCU anymore. If the caller is not within an RCU critical section and
2209 * does not hold the iothread lock, it must have other means of protecting the
2210 * pointer, such as a reference to the region that includes the incoming
2211 * ram_addr_t.
2212 */
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002213RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002214 ram_addr_t *offset)
pbrook5579c7f2009-04-11 14:47:08 +00002215{
pbrook94a6b542009-04-11 17:15:54 +00002216 RAMBlock *block;
2217 uint8_t *host = ptr;
2218
Jan Kiszka868bb332011-06-21 22:59:09 +02002219 if (xen_enabled()) {
Paolo Bonzinif615f392016-05-26 10:07:50 +02002220 ram_addr_t ram_addr;
Mike Day0dc3f442013-09-05 14:41:35 -04002221 rcu_read_lock();
Paolo Bonzinif615f392016-05-26 10:07:50 +02002222 ram_addr = xen_ram_addr_from_mapcache(ptr);
2223 block = qemu_get_ram_block(ram_addr);
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002224 if (block) {
Anthony PERARDd6b6aec2016-06-09 16:56:17 +01002225 *offset = ram_addr - block->offset;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002226 }
Mike Day0dc3f442013-09-05 14:41:35 -04002227 rcu_read_unlock();
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002228 return block;
Stefano Stabellini712c2b42011-05-19 18:35:46 +01002229 }
2230
Mike Day0dc3f442013-09-05 14:41:35 -04002231 rcu_read_lock();
2232 block = atomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002233 if (block && block->host && host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02002234 goto found;
2235 }
2236
Peter Xu99e15582017-05-12 12:17:39 +08002237 RAMBLOCK_FOREACH(block) {
Jun Nakajima432d2682010-08-31 16:41:25 +01002238 /* This case append when the block is not mapped. */
2239 if (block->host == NULL) {
2240 continue;
2241 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002242 if (host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02002243 goto found;
Alex Williamsonf471a172010-06-11 11:11:42 -06002244 }
pbrook94a6b542009-04-11 17:15:54 +00002245 }
Jun Nakajima432d2682010-08-31 16:41:25 +01002246
Mike Day0dc3f442013-09-05 14:41:35 -04002247 rcu_read_unlock();
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002248 return NULL;
Paolo Bonzini23887b72013-05-06 14:28:39 +02002249
2250found:
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002251 *offset = (host - block->host);
2252 if (round_offset) {
2253 *offset &= TARGET_PAGE_MASK;
2254 }
Mike Day0dc3f442013-09-05 14:41:35 -04002255 rcu_read_unlock();
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002256 return block;
2257}
2258
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002259/*
2260 * Finds the named RAMBlock
2261 *
2262 * name: The name of RAMBlock to find
2263 *
2264 * Returns: RAMBlock (or NULL if not found)
2265 */
2266RAMBlock *qemu_ram_block_by_name(const char *name)
2267{
2268 RAMBlock *block;
2269
Peter Xu99e15582017-05-12 12:17:39 +08002270 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002271 if (!strcmp(name, block->idstr)) {
2272 return block;
2273 }
2274 }
2275
2276 return NULL;
2277}
2278
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002279/* Some of the softmmu routines need to translate from a host pointer
2280 (typically a TLB entry) back to a ram offset. */
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002281ram_addr_t qemu_ram_addr_from_host(void *ptr)
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002282{
2283 RAMBlock *block;
Paolo Bonzinif615f392016-05-26 10:07:50 +02002284 ram_addr_t offset;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002285
Paolo Bonzinif615f392016-05-26 10:07:50 +02002286 block = qemu_ram_block_from_host(ptr, false, &offset);
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002287 if (!block) {
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002288 return RAM_ADDR_INVALID;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002289 }
2290
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002291 return block->offset + offset;
Marcelo Tosattie8902612010-10-11 15:31:19 -03002292}
Alex Williamsonf471a172010-06-11 11:11:42 -06002293
Paolo Bonzini49b24af2015-12-16 10:30:47 +01002294/* Called within RCU critical section. */
Avi Kivitya8170e52012-10-23 12:30:10 +02002295static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002296 uint64_t val, unsigned size)
bellard1ccde1c2004-02-06 19:46:14 +00002297{
Alex Bennéeba051fb2016-10-27 16:10:16 +01002298 bool locked = false;
2299
Juan Quintela52159192013-10-08 12:44:04 +02002300 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
Alex Bennéeba051fb2016-10-27 16:10:16 +01002301 locked = true;
2302 tb_lock();
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002303 tb_invalidate_phys_page_fast(ram_addr, size);
bellard3a7d9292005-08-21 09:26:42 +00002304 }
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002305 switch (size) {
2306 case 1:
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002307 stb_p(qemu_map_ram_ptr(NULL, ram_addr), val);
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002308 break;
2309 case 2:
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002310 stw_p(qemu_map_ram_ptr(NULL, ram_addr), val);
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002311 break;
2312 case 4:
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002313 stl_p(qemu_map_ram_ptr(NULL, ram_addr), val);
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002314 break;
2315 default:
2316 abort();
2317 }
Alex Bennéeba051fb2016-10-27 16:10:16 +01002318
2319 if (locked) {
2320 tb_unlock();
2321 }
2322
Paolo Bonzini58d27072015-03-23 11:56:01 +01002323 /* Set both VGA and migration bits for simplicity and to remove
2324 * the notdirty callback faster.
2325 */
2326 cpu_physical_memory_set_dirty_range(ram_addr, size,
2327 DIRTY_CLIENTS_NOCODE);
bellardf23db162005-08-21 19:12:28 +00002328 /* we remove the notdirty callback only if the code has been
2329 flushed */
Juan Quintelaa2cd8c82013-10-10 11:20:22 +02002330 if (!cpu_physical_memory_is_clean(ram_addr)) {
Peter Crosthwaitebcae01e2015-09-10 22:39:42 -07002331 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
Andreas Färber4917cf42013-05-27 05:17:50 +02002332 }
bellard1ccde1c2004-02-06 19:46:14 +00002333}
2334
Paolo Bonzinib018ddf2013-05-24 14:48:38 +02002335static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
2336 unsigned size, bool is_write)
2337{
2338 return is_write;
2339}
2340
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002341static const MemoryRegionOps notdirty_mem_ops = {
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002342 .write = notdirty_mem_write,
Paolo Bonzinib018ddf2013-05-24 14:48:38 +02002343 .valid.accepts = notdirty_mem_accepts,
Avi Kivity0e0df1e2012-01-02 00:32:15 +02002344 .endianness = DEVICE_NATIVE_ENDIAN,
bellard1ccde1c2004-02-06 19:46:14 +00002345};
2346
pbrook0f459d12008-06-09 00:20:13 +00002347/* Generate a debug exception if a watchpoint has been hit. */
Peter Maydell66b9b432015-04-26 16:49:24 +01002348static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
pbrook0f459d12008-06-09 00:20:13 +00002349{
Andreas Färber93afead2013-08-26 03:41:01 +02002350 CPUState *cpu = current_cpu;
Sergey Fedorov568496c2016-02-11 11:17:32 +00002351 CPUClass *cc = CPU_GET_CLASS(cpu);
Andreas Färber93afead2013-08-26 03:41:01 +02002352 CPUArchState *env = cpu->env_ptr;
aliguori06d55cc2008-11-18 20:24:06 +00002353 target_ulong pc, cs_base;
pbrook0f459d12008-06-09 00:20:13 +00002354 target_ulong vaddr;
aliguoria1d1bb32008-11-18 20:07:32 +00002355 CPUWatchpoint *wp;
Emilio G. Cota89fee742016-04-07 13:19:22 -04002356 uint32_t cpu_flags;
pbrook0f459d12008-06-09 00:20:13 +00002357
Andreas Färberff4700b2013-08-26 18:23:18 +02002358 if (cpu->watchpoint_hit) {
aliguori06d55cc2008-11-18 20:24:06 +00002359 /* We re-entered the check after replacing the TB. Now raise
2360 * the debug interrupt so that is will trigger after the
2361 * current instruction. */
Andreas Färber93afead2013-08-26 03:41:01 +02002362 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
aliguori06d55cc2008-11-18 20:24:06 +00002363 return;
2364 }
Andreas Färber93afead2013-08-26 03:41:01 +02002365 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
Julian Brown40612002017-02-07 18:29:59 +00002366 vaddr = cc->adjust_watchpoint_address(cpu, vaddr, len);
Andreas Färberff4700b2013-08-26 18:23:18 +02002367 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +01002368 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2369 && (wp->flags & flags)) {
Peter Maydell08225672014-09-12 14:06:48 +01002370 if (flags == BP_MEM_READ) {
2371 wp->flags |= BP_WATCHPOINT_HIT_READ;
2372 } else {
2373 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2374 }
2375 wp->hitaddr = vaddr;
Peter Maydell66b9b432015-04-26 16:49:24 +01002376 wp->hitattrs = attrs;
Andreas Färberff4700b2013-08-26 18:23:18 +02002377 if (!cpu->watchpoint_hit) {
Sergey Fedorov568496c2016-02-11 11:17:32 +00002378 if (wp->flags & BP_CPU &&
2379 !cc->debug_check_watchpoint(cpu, wp)) {
2380 wp->flags &= ~BP_WATCHPOINT_HIT;
2381 continue;
2382 }
Andreas Färberff4700b2013-08-26 18:23:18 +02002383 cpu->watchpoint_hit = wp;
KONRAD Frederica5e99822016-10-27 16:10:06 +01002384
Jan Kiszka8d04fb52017-02-23 18:29:11 +00002385 /* Both tb_lock and iothread_mutex will be reset when
2386 * cpu_loop_exit or cpu_loop_exit_noexc longjmp
2387 * back into the cpu_exec main loop.
KONRAD Frederica5e99822016-10-27 16:10:06 +01002388 */
2389 tb_lock();
Andreas Färber239c51a2013-09-01 17:12:23 +02002390 tb_check_watchpoint(cpu);
aliguori6e140f22008-11-18 20:37:55 +00002391 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
Andreas Färber27103422013-08-26 08:31:06 +02002392 cpu->exception_index = EXCP_DEBUG;
Andreas Färber5638d182013-08-27 17:52:12 +02002393 cpu_loop_exit(cpu);
aliguori6e140f22008-11-18 20:37:55 +00002394 } else {
2395 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
Andreas Färber648f0342013-09-01 17:43:17 +02002396 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
Peter Maydell6886b982016-05-17 15:18:04 +01002397 cpu_loop_exit_noexc(cpu);
aliguori6e140f22008-11-18 20:37:55 +00002398 }
aliguori06d55cc2008-11-18 20:24:06 +00002399 }
aliguori6e140f22008-11-18 20:37:55 +00002400 } else {
2401 wp->flags &= ~BP_WATCHPOINT_HIT;
pbrook0f459d12008-06-09 00:20:13 +00002402 }
2403 }
2404}
2405
pbrook6658ffb2007-03-16 23:58:11 +00002406/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2407 so these check for a hit then pass through to the normal out-of-line
2408 phys routines. */
Peter Maydell66b9b432015-04-26 16:49:24 +01002409static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2410 unsigned size, MemTxAttrs attrs)
pbrook6658ffb2007-03-16 23:58:11 +00002411{
Peter Maydell66b9b432015-04-26 16:49:24 +01002412 MemTxResult res;
2413 uint64_t data;
Peter Maydell79ed0412016-01-21 14:15:06 +00002414 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2415 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
pbrook6658ffb2007-03-16 23:58:11 +00002416
Peter Maydell66b9b432015-04-26 16:49:24 +01002417 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
Avi Kivity1ec9b902012-01-02 12:47:48 +02002418 switch (size) {
Max Filippov67364152012-01-29 00:01:40 +04002419 case 1:
Peter Maydell79ed0412016-01-21 14:15:06 +00002420 data = address_space_ldub(as, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04002421 break;
2422 case 2:
Peter Maydell79ed0412016-01-21 14:15:06 +00002423 data = address_space_lduw(as, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04002424 break;
2425 case 4:
Peter Maydell79ed0412016-01-21 14:15:06 +00002426 data = address_space_ldl(as, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04002427 break;
Avi Kivity1ec9b902012-01-02 12:47:48 +02002428 default: abort();
2429 }
Peter Maydell66b9b432015-04-26 16:49:24 +01002430 *pdata = data;
2431 return res;
2432}
2433
2434static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2435 uint64_t val, unsigned size,
2436 MemTxAttrs attrs)
2437{
2438 MemTxResult res;
Peter Maydell79ed0412016-01-21 14:15:06 +00002439 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2440 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
Peter Maydell66b9b432015-04-26 16:49:24 +01002441
2442 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2443 switch (size) {
2444 case 1:
Peter Maydell79ed0412016-01-21 14:15:06 +00002445 address_space_stb(as, addr, val, attrs, &res);
Peter Maydell66b9b432015-04-26 16:49:24 +01002446 break;
2447 case 2:
Peter Maydell79ed0412016-01-21 14:15:06 +00002448 address_space_stw(as, addr, val, attrs, &res);
Peter Maydell66b9b432015-04-26 16:49:24 +01002449 break;
2450 case 4:
Peter Maydell79ed0412016-01-21 14:15:06 +00002451 address_space_stl(as, addr, val, attrs, &res);
Peter Maydell66b9b432015-04-26 16:49:24 +01002452 break;
2453 default: abort();
2454 }
2455 return res;
pbrook6658ffb2007-03-16 23:58:11 +00002456}
2457
Avi Kivity1ec9b902012-01-02 12:47:48 +02002458static const MemoryRegionOps watch_mem_ops = {
Peter Maydell66b9b432015-04-26 16:49:24 +01002459 .read_with_attrs = watch_mem_read,
2460 .write_with_attrs = watch_mem_write,
Avi Kivity1ec9b902012-01-02 12:47:48 +02002461 .endianness = DEVICE_NATIVE_ENDIAN,
pbrook6658ffb2007-03-16 23:58:11 +00002462};
pbrook6658ffb2007-03-16 23:58:11 +00002463
Peter Maydellf25a49e2015-04-26 16:49:24 +01002464static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2465 unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00002466{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002467 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002468 uint8_t buf[8];
Peter Maydell5c9eb022015-04-26 16:49:24 +01002469 MemTxResult res;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002470
blueswir1db7b5422007-05-26 17:36:03 +00002471#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002472 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002473 subpage, len, addr);
blueswir1db7b5422007-05-26 17:36:03 +00002474#endif
Peter Maydell5c9eb022015-04-26 16:49:24 +01002475 res = address_space_read(subpage->as, addr + subpage->base,
2476 attrs, buf, len);
2477 if (res) {
2478 return res;
Peter Maydellf25a49e2015-04-26 16:49:24 +01002479 }
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002480 switch (len) {
2481 case 1:
Peter Maydellf25a49e2015-04-26 16:49:24 +01002482 *data = ldub_p(buf);
2483 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002484 case 2:
Peter Maydellf25a49e2015-04-26 16:49:24 +01002485 *data = lduw_p(buf);
2486 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002487 case 4:
Peter Maydellf25a49e2015-04-26 16:49:24 +01002488 *data = ldl_p(buf);
2489 return MEMTX_OK;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002490 case 8:
Peter Maydellf25a49e2015-04-26 16:49:24 +01002491 *data = ldq_p(buf);
2492 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002493 default:
2494 abort();
2495 }
blueswir1db7b5422007-05-26 17:36:03 +00002496}
2497
Peter Maydellf25a49e2015-04-26 16:49:24 +01002498static MemTxResult subpage_write(void *opaque, hwaddr addr,
2499 uint64_t value, unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00002500{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002501 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002502 uint8_t buf[8];
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002503
blueswir1db7b5422007-05-26 17:36:03 +00002504#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002505 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002506 " value %"PRIx64"\n",
2507 __func__, subpage, len, addr, value);
blueswir1db7b5422007-05-26 17:36:03 +00002508#endif
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002509 switch (len) {
2510 case 1:
2511 stb_p(buf, value);
2512 break;
2513 case 2:
2514 stw_p(buf, value);
2515 break;
2516 case 4:
2517 stl_p(buf, value);
2518 break;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002519 case 8:
2520 stq_p(buf, value);
2521 break;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002522 default:
2523 abort();
2524 }
Peter Maydell5c9eb022015-04-26 16:49:24 +01002525 return address_space_write(subpage->as, addr + subpage->base,
2526 attrs, buf, len);
blueswir1db7b5422007-05-26 17:36:03 +00002527}
2528
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002529static bool subpage_accepts(void *opaque, hwaddr addr,
Amos Kong016e9d62013-09-27 09:25:38 +08002530 unsigned len, bool is_write)
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002531{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002532 subpage_t *subpage = opaque;
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002533#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002534 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002535 __func__, subpage, is_write ? 'w' : 'r', len, addr);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002536#endif
2537
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002538 return address_space_access_valid(subpage->as, addr + subpage->base,
Amos Kong016e9d62013-09-27 09:25:38 +08002539 len, is_write);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002540}
2541
Avi Kivity70c68e42012-01-02 12:32:48 +02002542static const MemoryRegionOps subpage_ops = {
Peter Maydellf25a49e2015-04-26 16:49:24 +01002543 .read_with_attrs = subpage_read,
2544 .write_with_attrs = subpage_write,
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002545 .impl.min_access_size = 1,
2546 .impl.max_access_size = 8,
2547 .valid.min_access_size = 1,
2548 .valid.max_access_size = 8,
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002549 .valid.accepts = subpage_accepts,
Avi Kivity70c68e42012-01-02 12:32:48 +02002550 .endianness = DEVICE_NATIVE_ENDIAN,
blueswir1db7b5422007-05-26 17:36:03 +00002551};
2552
Anthony Liguoric227f092009-10-01 16:12:16 -05002553static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
Avi Kivity5312bd82012-02-12 18:32:55 +02002554 uint16_t section)
blueswir1db7b5422007-05-26 17:36:03 +00002555{
2556 int idx, eidx;
2557
2558 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2559 return -1;
2560 idx = SUBPAGE_IDX(start);
2561 eidx = SUBPAGE_IDX(end);
2562#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002563 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2564 __func__, mmio, start, end, idx, eidx, section);
blueswir1db7b5422007-05-26 17:36:03 +00002565#endif
blueswir1db7b5422007-05-26 17:36:03 +00002566 for (; idx <= eidx; idx++) {
Avi Kivity5312bd82012-02-12 18:32:55 +02002567 mmio->sub_section[idx] = section;
blueswir1db7b5422007-05-26 17:36:03 +00002568 }
2569
2570 return 0;
2571}
2572
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002573static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
blueswir1db7b5422007-05-26 17:36:03 +00002574{
Anthony Liguoric227f092009-10-01 16:12:16 -05002575 subpage_t *mmio;
blueswir1db7b5422007-05-26 17:36:03 +00002576
Vijaya Kumar K2615fab2016-10-24 16:26:49 +01002577 mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002578 mmio->as = as;
aliguori1eec6142009-02-05 22:06:18 +00002579 mmio->base = base;
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002580 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07002581 NULL, TARGET_PAGE_SIZE);
Avi Kivityb3b00c72012-01-02 13:20:11 +02002582 mmio->iomem.subpage = true;
blueswir1db7b5422007-05-26 17:36:03 +00002583#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002584 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2585 mmio, base, TARGET_PAGE_SIZE);
blueswir1db7b5422007-05-26 17:36:03 +00002586#endif
Liu Ping Fanb41aac42013-05-29 11:09:17 +02002587 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
blueswir1db7b5422007-05-26 17:36:03 +00002588
2589 return mmio;
2590}
2591
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002592static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2593 MemoryRegion *mr)
Avi Kivity5312bd82012-02-12 18:32:55 +02002594{
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002595 assert(as);
Avi Kivity5312bd82012-02-12 18:32:55 +02002596 MemoryRegionSection section = {
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002597 .address_space = as,
Avi Kivity5312bd82012-02-12 18:32:55 +02002598 .mr = mr,
2599 .offset_within_address_space = 0,
2600 .offset_within_region = 0,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02002601 .size = int128_2_64(),
Avi Kivity5312bd82012-02-12 18:32:55 +02002602 };
2603
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002604 return phys_section_add(map, &section);
Avi Kivity5312bd82012-02-12 18:32:55 +02002605}
2606
Peter Maydella54c87b2016-01-21 14:15:05 +00002607MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
Avi Kivityaa102232012-03-08 17:06:55 +02002608{
Peter Maydella54c87b2016-01-21 14:15:05 +00002609 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2610 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
Peter Maydell32857f42015-10-01 15:29:50 +01002611 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002612 MemoryRegionSection *sections = d->map.sections;
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +02002613
2614 return sections[index & ~TARGET_PAGE_MASK].mr;
Avi Kivityaa102232012-03-08 17:06:55 +02002615}
2616
Avi Kivitye9179ce2009-06-14 11:38:52 +03002617static void io_mem_init(void)
2618{
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002619 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002620 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002621 NULL, UINT64_MAX);
Jan Kiszka8d04fb52017-02-23 18:29:11 +00002622
2623 /* io_mem_notdirty calls tb_invalidate_phys_page_fast,
2624 * which can be called without the iothread mutex.
2625 */
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002626 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002627 NULL, UINT64_MAX);
Jan Kiszka8d04fb52017-02-23 18:29:11 +00002628 memory_region_clear_global_locking(&io_mem_notdirty);
2629
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002630 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002631 NULL, UINT64_MAX);
Avi Kivitye9179ce2009-06-14 11:38:52 +03002632}
2633
Avi Kivityac1970f2012-10-03 16:22:53 +02002634static void mem_begin(MemoryListener *listener)
2635{
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002636 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002637 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2638 uint16_t n;
2639
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002640 n = dummy_section(&d->map, as, &io_mem_unassigned);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002641 assert(n == PHYS_SECTION_UNASSIGNED);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002642 n = dummy_section(&d->map, as, &io_mem_notdirty);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002643 assert(n == PHYS_SECTION_NOTDIRTY);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002644 n = dummy_section(&d->map, as, &io_mem_rom);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002645 assert(n == PHYS_SECTION_ROM);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002646 n = dummy_section(&d->map, as, &io_mem_watch);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002647 assert(n == PHYS_SECTION_WATCH);
Paolo Bonzini00752702013-05-29 12:13:54 +02002648
Michael S. Tsirkin9736e552013-11-11 14:42:43 +02002649 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
Paolo Bonzini00752702013-05-29 12:13:54 +02002650 d->as = as;
2651 as->next_dispatch = d;
2652}
2653
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002654static void address_space_dispatch_free(AddressSpaceDispatch *d)
2655{
2656 phys_sections_free(&d->map);
2657 g_free(d);
2658}
2659
Paolo Bonzini00752702013-05-29 12:13:54 +02002660static void mem_commit(MemoryListener *listener)
2661{
2662 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Paolo Bonzini0475d942013-05-29 12:28:21 +02002663 AddressSpaceDispatch *cur = as->dispatch;
2664 AddressSpaceDispatch *next = as->next_dispatch;
Avi Kivityac1970f2012-10-03 16:22:53 +02002665
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002666 phys_page_compact_all(next, next->map.nodes_nb);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +02002667
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002668 atomic_rcu_set(&as->dispatch, next);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002669 if (cur) {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002670 call_rcu(cur, address_space_dispatch_free, rcu);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002671 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02002672}
2673
Avi Kivity1d711482012-10-02 18:54:45 +02002674static void tcg_commit(MemoryListener *listener)
Avi Kivity50c1e142012-02-08 21:36:02 +02002675{
Peter Maydell32857f42015-10-01 15:29:50 +01002676 CPUAddressSpace *cpuas;
2677 AddressSpaceDispatch *d;
Avi Kivity117712c2012-02-12 21:23:17 +02002678
2679 /* since each CPU stores ram addresses in its TLB cache, we must
2680 reset the modified entries */
Peter Maydell32857f42015-10-01 15:29:50 +01002681 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2682 cpu_reloading_memory_map();
2683 /* The CPU and TLB are protected by the iothread lock.
2684 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2685 * may have split the RCU critical section.
2686 */
2687 d = atomic_rcu_read(&cpuas->as->dispatch);
Alex Bennéef35e44e2016-10-21 16:34:18 +01002688 atomic_rcu_set(&cpuas->memory_dispatch, d);
Alex Bennéed10eb082016-11-14 14:17:28 +00002689 tlb_flush(cpuas->cpu);
Avi Kivity50c1e142012-02-08 21:36:02 +02002690}
2691
Avi Kivityac1970f2012-10-03 16:22:53 +02002692void address_space_init_dispatch(AddressSpace *as)
2693{
Paolo Bonzini00752702013-05-29 12:13:54 +02002694 as->dispatch = NULL;
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002695 as->dispatch_listener = (MemoryListener) {
Avi Kivityac1970f2012-10-03 16:22:53 +02002696 .begin = mem_begin,
Paolo Bonzini00752702013-05-29 12:13:54 +02002697 .commit = mem_commit,
Avi Kivityac1970f2012-10-03 16:22:53 +02002698 .region_add = mem_add,
2699 .region_nop = mem_add,
2700 .priority = 0,
2701 };
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002702 memory_listener_register(&as->dispatch_listener, as);
Avi Kivityac1970f2012-10-03 16:22:53 +02002703}
2704
Paolo Bonzini6e48e8f2015-02-10 10:25:44 -07002705void address_space_unregister(AddressSpace *as)
2706{
2707 memory_listener_unregister(&as->dispatch_listener);
2708}
2709
Avi Kivity83f3c252012-10-07 12:59:55 +02002710void address_space_destroy_dispatch(AddressSpace *as)
2711{
2712 AddressSpaceDispatch *d = as->dispatch;
2713
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002714 atomic_rcu_set(&as->dispatch, NULL);
2715 if (d) {
2716 call_rcu(d, address_space_dispatch_free, rcu);
2717 }
Avi Kivity83f3c252012-10-07 12:59:55 +02002718}
2719
Avi Kivity62152b82011-07-26 14:26:14 +03002720static void memory_map_init(void)
2721{
Anthony Liguori7267c092011-08-20 22:09:37 -05002722 system_memory = g_malloc(sizeof(*system_memory));
Paolo Bonzini03f49952013-11-07 17:14:36 +01002723
Paolo Bonzini57271d62013-11-07 17:14:37 +01002724 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002725 address_space_init(&address_space_memory, system_memory, "memory");
Avi Kivity309cb472011-08-08 16:09:03 +03002726
Anthony Liguori7267c092011-08-20 22:09:37 -05002727 system_io = g_malloc(sizeof(*system_io));
Jan Kiszka3bb28b72013-09-02 18:43:30 +02002728 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2729 65536);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002730 address_space_init(&address_space_io, system_io, "I/O");
Avi Kivity62152b82011-07-26 14:26:14 +03002731}
2732
2733MemoryRegion *get_system_memory(void)
2734{
2735 return system_memory;
2736}
2737
Avi Kivity309cb472011-08-08 16:09:03 +03002738MemoryRegion *get_system_io(void)
2739{
2740 return system_io;
2741}
2742
pbrooke2eef172008-06-08 01:09:01 +00002743#endif /* !defined(CONFIG_USER_ONLY) */
2744
bellard13eb76e2004-01-24 15:23:36 +00002745/* physical memory access (slow version, mainly for debug) */
2746#if defined(CONFIG_USER_ONLY)
Andreas Färberf17ec442013-06-29 19:40:58 +02002747int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
Paul Brooka68fe892010-03-01 00:08:59 +00002748 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00002749{
2750 int l, flags;
2751 target_ulong page;
pbrook53a59602006-03-25 19:31:22 +00002752 void * p;
bellard13eb76e2004-01-24 15:23:36 +00002753
2754 while (len > 0) {
2755 page = addr & TARGET_PAGE_MASK;
2756 l = (page + TARGET_PAGE_SIZE) - addr;
2757 if (l > len)
2758 l = len;
2759 flags = page_get_flags(page);
2760 if (!(flags & PAGE_VALID))
Paul Brooka68fe892010-03-01 00:08:59 +00002761 return -1;
bellard13eb76e2004-01-24 15:23:36 +00002762 if (is_write) {
2763 if (!(flags & PAGE_WRITE))
Paul Brooka68fe892010-03-01 00:08:59 +00002764 return -1;
bellard579a97f2007-11-11 14:26:47 +00002765 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00002766 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
Paul Brooka68fe892010-03-01 00:08:59 +00002767 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00002768 memcpy(p, buf, l);
2769 unlock_user(p, addr, l);
bellard13eb76e2004-01-24 15:23:36 +00002770 } else {
2771 if (!(flags & PAGE_READ))
Paul Brooka68fe892010-03-01 00:08:59 +00002772 return -1;
bellard579a97f2007-11-11 14:26:47 +00002773 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00002774 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
Paul Brooka68fe892010-03-01 00:08:59 +00002775 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00002776 memcpy(buf, p, l);
aurel325b257572008-04-28 08:54:59 +00002777 unlock_user(p, addr, 0);
bellard13eb76e2004-01-24 15:23:36 +00002778 }
2779 len -= l;
2780 buf += l;
2781 addr += l;
2782 }
Paul Brooka68fe892010-03-01 00:08:59 +00002783 return 0;
bellard13eb76e2004-01-24 15:23:36 +00002784}
bellard8df1cd02005-01-28 22:37:22 +00002785
bellard13eb76e2004-01-24 15:23:36 +00002786#else
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002787
Paolo Bonzini845b6212015-03-23 11:45:53 +01002788static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
Avi Kivitya8170e52012-10-23 12:30:10 +02002789 hwaddr length)
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002790{
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002791 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002792 addr += memory_region_get_ram_addr(mr);
2793
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002794 /* No early return if dirty_log_mask is or becomes 0, because
2795 * cpu_physical_memory_set_dirty_range will still call
2796 * xen_modified_memory.
2797 */
2798 if (dirty_log_mask) {
2799 dirty_log_mask =
2800 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002801 }
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002802 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
Alex Bennéeba051fb2016-10-27 16:10:16 +01002803 tb_lock();
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002804 tb_invalidate_phys_range(addr, addr + length);
Alex Bennéeba051fb2016-10-27 16:10:16 +01002805 tb_unlock();
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002806 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2807 }
2808 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002809}
2810
Richard Henderson23326162013-07-08 14:55:59 -07002811static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
Paolo Bonzini82f25632013-05-24 11:59:43 +02002812{
Paolo Bonzinie1622f42013-07-17 13:17:41 +02002813 unsigned access_size_max = mr->ops->valid.max_access_size;
Richard Henderson23326162013-07-08 14:55:59 -07002814
2815 /* Regions are assumed to support 1-4 byte accesses unless
2816 otherwise specified. */
Richard Henderson23326162013-07-08 14:55:59 -07002817 if (access_size_max == 0) {
2818 access_size_max = 4;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002819 }
Richard Henderson23326162013-07-08 14:55:59 -07002820
2821 /* Bound the maximum access by the alignment of the address. */
2822 if (!mr->ops->impl.unaligned) {
2823 unsigned align_size_max = addr & -addr;
2824 if (align_size_max != 0 && align_size_max < access_size_max) {
2825 access_size_max = align_size_max;
2826 }
2827 }
2828
2829 /* Don't attempt accesses larger than the maximum. */
2830 if (l > access_size_max) {
2831 l = access_size_max;
2832 }
Peter Maydell6554f5c2015-07-24 13:33:10 +01002833 l = pow2floor(l);
Richard Henderson23326162013-07-08 14:55:59 -07002834
2835 return l;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002836}
2837
Jan Kiszka4840f102015-06-18 18:47:22 +02002838static bool prepare_mmio_access(MemoryRegion *mr)
Paolo Bonzini125b3802015-06-18 18:47:21 +02002839{
Jan Kiszka4840f102015-06-18 18:47:22 +02002840 bool unlocked = !qemu_mutex_iothread_locked();
2841 bool release_lock = false;
2842
2843 if (unlocked && mr->global_locking) {
2844 qemu_mutex_lock_iothread();
2845 unlocked = false;
2846 release_lock = true;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002847 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002848 if (mr->flush_coalesced_mmio) {
2849 if (unlocked) {
2850 qemu_mutex_lock_iothread();
2851 }
2852 qemu_flush_coalesced_mmio_buffer();
2853 if (unlocked) {
2854 qemu_mutex_unlock_iothread();
2855 }
2856 }
2857
2858 return release_lock;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002859}
2860
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002861/* Called within RCU critical section. */
2862static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
2863 MemTxAttrs attrs,
2864 const uint8_t *buf,
2865 int len, hwaddr addr1,
2866 hwaddr l, MemoryRegion *mr)
bellard13eb76e2004-01-24 15:23:36 +00002867{
bellard13eb76e2004-01-24 15:23:36 +00002868 uint8_t *ptr;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002869 uint64_t val;
Peter Maydell3b643492015-04-26 16:49:23 +01002870 MemTxResult result = MEMTX_OK;
Jan Kiszka4840f102015-06-18 18:47:22 +02002871 bool release_lock = false;
ths3b46e622007-09-17 08:09:54 +00002872
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002873 for (;;) {
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002874 if (!memory_access_is_direct(mr, true)) {
2875 release_lock |= prepare_mmio_access(mr);
2876 l = memory_access_size(mr, l, addr1);
2877 /* XXX: could force current_cpu to NULL to avoid
2878 potential bugs */
2879 switch (l) {
2880 case 8:
2881 /* 64 bit write access */
2882 val = ldq_p(buf);
2883 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2884 attrs);
2885 break;
2886 case 4:
2887 /* 32 bit write access */
Ladi Prosek6da67de2017-01-26 15:22:37 +01002888 val = (uint32_t)ldl_p(buf);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002889 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2890 attrs);
2891 break;
2892 case 2:
2893 /* 16 bit write access */
2894 val = lduw_p(buf);
2895 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2896 attrs);
2897 break;
2898 case 1:
2899 /* 8 bit write access */
2900 val = ldub_p(buf);
2901 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2902 attrs);
2903 break;
2904 default:
2905 abort();
bellard13eb76e2004-01-24 15:23:36 +00002906 }
2907 } else {
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002908 /* RAM case */
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002909 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002910 memcpy(ptr, buf, l);
2911 invalidate_and_set_dirty(mr, addr1, l);
bellard13eb76e2004-01-24 15:23:36 +00002912 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002913
2914 if (release_lock) {
2915 qemu_mutex_unlock_iothread();
2916 release_lock = false;
2917 }
2918
bellard13eb76e2004-01-24 15:23:36 +00002919 len -= l;
2920 buf += l;
2921 addr += l;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002922
2923 if (!len) {
2924 break;
2925 }
2926
2927 l = len;
2928 mr = address_space_translate(as, addr, &addr1, &l, true);
bellard13eb76e2004-01-24 15:23:36 +00002929 }
Paolo Bonzinifd8aaa72013-05-21 09:56:55 +02002930
Peter Maydell3b643492015-04-26 16:49:23 +01002931 return result;
bellard13eb76e2004-01-24 15:23:36 +00002932}
bellard8df1cd02005-01-28 22:37:22 +00002933
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002934MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2935 const uint8_t *buf, int len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002936{
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002937 hwaddr l;
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002938 hwaddr addr1;
2939 MemoryRegion *mr;
2940 MemTxResult result = MEMTX_OK;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002941
2942 if (len > 0) {
2943 rcu_read_lock();
2944 l = len;
2945 mr = address_space_translate(as, addr, &addr1, &l, true);
2946 result = address_space_write_continue(as, addr, attrs, buf, len,
2947 addr1, l, mr);
2948 rcu_read_unlock();
2949 }
2950
2951 return result;
2952}
2953
2954/* Called within RCU critical section. */
2955MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
2956 MemTxAttrs attrs, uint8_t *buf,
2957 int len, hwaddr addr1, hwaddr l,
2958 MemoryRegion *mr)
2959{
2960 uint8_t *ptr;
2961 uint64_t val;
2962 MemTxResult result = MEMTX_OK;
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002963 bool release_lock = false;
2964
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002965 for (;;) {
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002966 if (!memory_access_is_direct(mr, false)) {
2967 /* I/O case */
2968 release_lock |= prepare_mmio_access(mr);
2969 l = memory_access_size(mr, l, addr1);
2970 switch (l) {
2971 case 8:
2972 /* 64 bit read access */
2973 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2974 attrs);
2975 stq_p(buf, val);
2976 break;
2977 case 4:
2978 /* 32 bit read access */
2979 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2980 attrs);
2981 stl_p(buf, val);
2982 break;
2983 case 2:
2984 /* 16 bit read access */
2985 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2986 attrs);
2987 stw_p(buf, val);
2988 break;
2989 case 1:
2990 /* 8 bit read access */
2991 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2992 attrs);
2993 stb_p(buf, val);
2994 break;
2995 default:
2996 abort();
2997 }
2998 } else {
2999 /* RAM case */
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01003000 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01003001 memcpy(buf, ptr, l);
3002 }
3003
3004 if (release_lock) {
3005 qemu_mutex_unlock_iothread();
3006 release_lock = false;
3007 }
3008
3009 len -= l;
3010 buf += l;
3011 addr += l;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01003012
3013 if (!len) {
3014 break;
3015 }
3016
3017 l = len;
3018 mr = address_space_translate(as, addr, &addr1, &l, false);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01003019 }
Paolo Bonzinia203ac72015-12-09 10:18:57 +01003020
3021 return result;
3022}
3023
Paolo Bonzini3cc8f882015-12-09 10:34:13 +01003024MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
3025 MemTxAttrs attrs, uint8_t *buf, int len)
Paolo Bonzinia203ac72015-12-09 10:18:57 +01003026{
3027 hwaddr l;
3028 hwaddr addr1;
3029 MemoryRegion *mr;
3030 MemTxResult result = MEMTX_OK;
3031
3032 if (len > 0) {
3033 rcu_read_lock();
3034 l = len;
3035 mr = address_space_translate(as, addr, &addr1, &l, false);
3036 result = address_space_read_continue(as, addr, attrs, buf, len,
3037 addr1, l, mr);
3038 rcu_read_unlock();
3039 }
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01003040
3041 return result;
Avi Kivityac1970f2012-10-03 16:22:53 +02003042}
3043
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01003044MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
3045 uint8_t *buf, int len, bool is_write)
3046{
3047 if (is_write) {
3048 return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
3049 } else {
3050 return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
3051 }
3052}
Avi Kivityac1970f2012-10-03 16:22:53 +02003053
Avi Kivitya8170e52012-10-23 12:30:10 +02003054void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
Avi Kivityac1970f2012-10-03 16:22:53 +02003055 int len, int is_write)
3056{
Peter Maydell5c9eb022015-04-26 16:49:24 +01003057 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
3058 buf, len, is_write);
Avi Kivityac1970f2012-10-03 16:22:53 +02003059}
3060
Alexander Graf582b55a2013-12-11 14:17:44 +01003061enum write_rom_type {
3062 WRITE_DATA,
3063 FLUSH_CACHE,
3064};
3065
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10003066static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
Alexander Graf582b55a2013-12-11 14:17:44 +01003067 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
bellardd0ecd2a2006-04-23 17:14:48 +00003068{
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003069 hwaddr l;
bellardd0ecd2a2006-04-23 17:14:48 +00003070 uint8_t *ptr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003071 hwaddr addr1;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003072 MemoryRegion *mr;
ths3b46e622007-09-17 08:09:54 +00003073
Paolo Bonzini41063e12015-03-18 14:21:43 +01003074 rcu_read_lock();
bellardd0ecd2a2006-04-23 17:14:48 +00003075 while (len > 0) {
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003076 l = len;
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10003077 mr = address_space_translate(as, addr, &addr1, &l, true);
ths3b46e622007-09-17 08:09:54 +00003078
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003079 if (!(memory_region_is_ram(mr) ||
3080 memory_region_is_romd(mr))) {
Paolo Bonzinib242e0e2015-07-04 00:24:51 +02003081 l = memory_access_size(mr, l, addr1);
bellardd0ecd2a2006-04-23 17:14:48 +00003082 } else {
bellardd0ecd2a2006-04-23 17:14:48 +00003083 /* ROM/RAM case */
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01003084 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
Alexander Graf582b55a2013-12-11 14:17:44 +01003085 switch (type) {
3086 case WRITE_DATA:
3087 memcpy(ptr, buf, l);
Paolo Bonzini845b6212015-03-23 11:45:53 +01003088 invalidate_and_set_dirty(mr, addr1, l);
Alexander Graf582b55a2013-12-11 14:17:44 +01003089 break;
3090 case FLUSH_CACHE:
3091 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
3092 break;
3093 }
bellardd0ecd2a2006-04-23 17:14:48 +00003094 }
3095 len -= l;
3096 buf += l;
3097 addr += l;
3098 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003099 rcu_read_unlock();
bellardd0ecd2a2006-04-23 17:14:48 +00003100}
3101
Alexander Graf582b55a2013-12-11 14:17:44 +01003102/* used for ROM loading : can write in RAM and ROM */
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10003103void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
Alexander Graf582b55a2013-12-11 14:17:44 +01003104 const uint8_t *buf, int len)
3105{
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10003106 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
Alexander Graf582b55a2013-12-11 14:17:44 +01003107}
3108
3109void cpu_flush_icache_range(hwaddr start, int len)
3110{
3111 /*
3112 * This function should do the same thing as an icache flush that was
3113 * triggered from within the guest. For TCG we are always cache coherent,
3114 * so there is no need to flush anything. For KVM / Xen we need to flush
3115 * the host's instruction cache at least.
3116 */
3117 if (tcg_enabled()) {
3118 return;
3119 }
3120
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10003121 cpu_physical_memory_write_rom_internal(&address_space_memory,
3122 start, NULL, len, FLUSH_CACHE);
Alexander Graf582b55a2013-12-11 14:17:44 +01003123}
3124
aliguori6d16c2f2009-01-22 16:59:11 +00003125typedef struct {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003126 MemoryRegion *mr;
aliguori6d16c2f2009-01-22 16:59:11 +00003127 void *buffer;
Avi Kivitya8170e52012-10-23 12:30:10 +02003128 hwaddr addr;
3129 hwaddr len;
Fam Zhengc2cba0f2015-03-16 17:03:33 +08003130 bool in_use;
aliguori6d16c2f2009-01-22 16:59:11 +00003131} BounceBuffer;
3132
3133static BounceBuffer bounce;
3134
aliguoriba223c22009-01-22 16:59:16 +00003135typedef struct MapClient {
Fam Zhenge95205e2015-03-16 17:03:37 +08003136 QEMUBH *bh;
Blue Swirl72cf2d42009-09-12 07:36:22 +00003137 QLIST_ENTRY(MapClient) link;
aliguoriba223c22009-01-22 16:59:16 +00003138} MapClient;
3139
Fam Zheng38e047b2015-03-16 17:03:35 +08003140QemuMutex map_client_list_lock;
Blue Swirl72cf2d42009-09-12 07:36:22 +00003141static QLIST_HEAD(map_client_list, MapClient) map_client_list
3142 = QLIST_HEAD_INITIALIZER(map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00003143
Fam Zhenge95205e2015-03-16 17:03:37 +08003144static void cpu_unregister_map_client_do(MapClient *client)
aliguoriba223c22009-01-22 16:59:16 +00003145{
Blue Swirl72cf2d42009-09-12 07:36:22 +00003146 QLIST_REMOVE(client, link);
Anthony Liguori7267c092011-08-20 22:09:37 -05003147 g_free(client);
aliguoriba223c22009-01-22 16:59:16 +00003148}
3149
Fam Zheng33b6c2e2015-03-16 17:03:36 +08003150static void cpu_notify_map_clients_locked(void)
aliguoriba223c22009-01-22 16:59:16 +00003151{
3152 MapClient *client;
3153
Blue Swirl72cf2d42009-09-12 07:36:22 +00003154 while (!QLIST_EMPTY(&map_client_list)) {
3155 client = QLIST_FIRST(&map_client_list);
Fam Zhenge95205e2015-03-16 17:03:37 +08003156 qemu_bh_schedule(client->bh);
3157 cpu_unregister_map_client_do(client);
aliguoriba223c22009-01-22 16:59:16 +00003158 }
3159}
3160
Fam Zhenge95205e2015-03-16 17:03:37 +08003161void cpu_register_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00003162{
3163 MapClient *client = g_malloc(sizeof(*client));
3164
Fam Zheng38e047b2015-03-16 17:03:35 +08003165 qemu_mutex_lock(&map_client_list_lock);
Fam Zhenge95205e2015-03-16 17:03:37 +08003166 client->bh = bh;
bellardd0ecd2a2006-04-23 17:14:48 +00003167 QLIST_INSERT_HEAD(&map_client_list, client, link);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08003168 if (!atomic_read(&bounce.in_use)) {
3169 cpu_notify_map_clients_locked();
3170 }
Fam Zheng38e047b2015-03-16 17:03:35 +08003171 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00003172}
3173
Fam Zheng38e047b2015-03-16 17:03:35 +08003174void cpu_exec_init_all(void)
3175{
3176 qemu_mutex_init(&ram_list.mutex);
Peter Maydell20bccb82016-10-24 16:26:49 +01003177 /* The data structures we set up here depend on knowing the page size,
3178 * so no more changes can be made after this point.
3179 * In an ideal world, nothing we did before we had finished the
3180 * machine setup would care about the target page size, and we could
3181 * do this much later, rather than requiring board models to state
3182 * up front what their requirements are.
3183 */
3184 finalize_target_page_bits();
Fam Zheng38e047b2015-03-16 17:03:35 +08003185 io_mem_init();
Paolo Bonzini680a4782015-11-02 09:23:52 +01003186 memory_map_init();
Fam Zheng38e047b2015-03-16 17:03:35 +08003187 qemu_mutex_init(&map_client_list_lock);
3188}
3189
Fam Zhenge95205e2015-03-16 17:03:37 +08003190void cpu_unregister_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00003191{
Fam Zhenge95205e2015-03-16 17:03:37 +08003192 MapClient *client;
bellardd0ecd2a2006-04-23 17:14:48 +00003193
Fam Zhenge95205e2015-03-16 17:03:37 +08003194 qemu_mutex_lock(&map_client_list_lock);
3195 QLIST_FOREACH(client, &map_client_list, link) {
3196 if (client->bh == bh) {
3197 cpu_unregister_map_client_do(client);
3198 break;
3199 }
3200 }
3201 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00003202}
3203
3204static void cpu_notify_map_clients(void)
3205{
Fam Zheng38e047b2015-03-16 17:03:35 +08003206 qemu_mutex_lock(&map_client_list_lock);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08003207 cpu_notify_map_clients_locked();
Fam Zheng38e047b2015-03-16 17:03:35 +08003208 qemu_mutex_unlock(&map_client_list_lock);
aliguori6d16c2f2009-01-22 16:59:11 +00003209}
3210
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003211bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
3212{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003213 MemoryRegion *mr;
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003214 hwaddr l, xlat;
3215
Paolo Bonzini41063e12015-03-18 14:21:43 +01003216 rcu_read_lock();
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003217 while (len > 0) {
3218 l = len;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003219 mr = address_space_translate(as, addr, &xlat, &l, is_write);
3220 if (!memory_access_is_direct(mr, is_write)) {
3221 l = memory_access_size(mr, l, addr);
3222 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
Roman Kapl5ad4a2b2017-01-09 12:09:21 +01003223 rcu_read_unlock();
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003224 return false;
3225 }
3226 }
3227
3228 len -= l;
3229 addr += l;
3230 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003231 rcu_read_unlock();
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003232 return true;
3233}
3234
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003235static hwaddr
3236address_space_extend_translation(AddressSpace *as, hwaddr addr, hwaddr target_len,
3237 MemoryRegion *mr, hwaddr base, hwaddr len,
3238 bool is_write)
3239{
3240 hwaddr done = 0;
3241 hwaddr xlat;
3242 MemoryRegion *this_mr;
3243
3244 for (;;) {
3245 target_len -= len;
3246 addr += len;
3247 done += len;
3248 if (target_len == 0) {
3249 return done;
3250 }
3251
3252 len = target_len;
3253 this_mr = address_space_translate(as, addr, &xlat, &len, is_write);
3254 if (this_mr != mr || xlat != base + done) {
3255 return done;
3256 }
3257 }
3258}
3259
aliguori6d16c2f2009-01-22 16:59:11 +00003260/* Map a physical memory region into a host virtual address.
3261 * May map a subset of the requested range, given by and returned in *plen.
3262 * May return NULL if resources needed to perform the mapping are exhausted.
3263 * Use only for reads OR writes - not for read-modify-write operations.
aliguoriba223c22009-01-22 16:59:16 +00003264 * Use cpu_register_map_client() to know when retrying the map operation is
3265 * likely to succeed.
aliguori6d16c2f2009-01-22 16:59:11 +00003266 */
Avi Kivityac1970f2012-10-03 16:22:53 +02003267void *address_space_map(AddressSpace *as,
Avi Kivitya8170e52012-10-23 12:30:10 +02003268 hwaddr addr,
3269 hwaddr *plen,
Avi Kivityac1970f2012-10-03 16:22:53 +02003270 bool is_write)
aliguori6d16c2f2009-01-22 16:59:11 +00003271{
Avi Kivitya8170e52012-10-23 12:30:10 +02003272 hwaddr len = *plen;
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003273 hwaddr l, xlat;
3274 MemoryRegion *mr;
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01003275 void *ptr;
aliguori6d16c2f2009-01-22 16:59:11 +00003276
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003277 if (len == 0) {
3278 return NULL;
3279 }
aliguori6d16c2f2009-01-22 16:59:11 +00003280
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003281 l = len;
Paolo Bonzini41063e12015-03-18 14:21:43 +01003282 rcu_read_lock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003283 mr = address_space_translate(as, addr, &xlat, &l, is_write);
Paolo Bonzini41063e12015-03-18 14:21:43 +01003284
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003285 if (!memory_access_is_direct(mr, is_write)) {
Fam Zhengc2cba0f2015-03-16 17:03:33 +08003286 if (atomic_xchg(&bounce.in_use, true)) {
Paolo Bonzini41063e12015-03-18 14:21:43 +01003287 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003288 return NULL;
aliguori6d16c2f2009-01-22 16:59:11 +00003289 }
Kevin Wolfe85d9db2013-07-22 14:30:23 +02003290 /* Avoid unbounded allocations */
3291 l = MIN(l, TARGET_PAGE_SIZE);
3292 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003293 bounce.addr = addr;
3294 bounce.len = l;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003295
3296 memory_region_ref(mr);
3297 bounce.mr = mr;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003298 if (!is_write) {
Peter Maydell5c9eb022015-04-26 16:49:24 +01003299 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
3300 bounce.buffer, l);
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01003301 }
aliguori6d16c2f2009-01-22 16:59:11 +00003302
Paolo Bonzini41063e12015-03-18 14:21:43 +01003303 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003304 *plen = l;
3305 return bounce.buffer;
3306 }
3307
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003308
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003309 memory_region_ref(mr);
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003310 *plen = address_space_extend_translation(as, addr, len, mr, xlat, l, is_write);
3311 ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen);
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01003312 rcu_read_unlock();
3313
3314 return ptr;
aliguori6d16c2f2009-01-22 16:59:11 +00003315}
3316
Avi Kivityac1970f2012-10-03 16:22:53 +02003317/* Unmaps a memory region previously mapped by address_space_map().
aliguori6d16c2f2009-01-22 16:59:11 +00003318 * Will also mark the memory as dirty if is_write == 1. access_len gives
3319 * the amount of memory that was actually read or written by the caller.
3320 */
Avi Kivitya8170e52012-10-23 12:30:10 +02003321void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3322 int is_write, hwaddr access_len)
aliguori6d16c2f2009-01-22 16:59:11 +00003323{
3324 if (buffer != bounce.buffer) {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003325 MemoryRegion *mr;
3326 ram_addr_t addr1;
3327
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01003328 mr = memory_region_from_host(buffer, &addr1);
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003329 assert(mr != NULL);
aliguori6d16c2f2009-01-22 16:59:11 +00003330 if (is_write) {
Paolo Bonzini845b6212015-03-23 11:45:53 +01003331 invalidate_and_set_dirty(mr, addr1, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00003332 }
Jan Kiszka868bb332011-06-21 22:59:09 +02003333 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003334 xen_invalidate_map_cache_entry(buffer);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01003335 }
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003336 memory_region_unref(mr);
aliguori6d16c2f2009-01-22 16:59:11 +00003337 return;
3338 }
3339 if (is_write) {
Peter Maydell5c9eb022015-04-26 16:49:24 +01003340 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
3341 bounce.buffer, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00003342 }
Herve Poussineauf8a83242010-01-24 21:23:56 +00003343 qemu_vfree(bounce.buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00003344 bounce.buffer = NULL;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003345 memory_region_unref(bounce.mr);
Fam Zhengc2cba0f2015-03-16 17:03:33 +08003346 atomic_mb_set(&bounce.in_use, false);
aliguoriba223c22009-01-22 16:59:16 +00003347 cpu_notify_map_clients();
aliguori6d16c2f2009-01-22 16:59:11 +00003348}
bellardd0ecd2a2006-04-23 17:14:48 +00003349
Avi Kivitya8170e52012-10-23 12:30:10 +02003350void *cpu_physical_memory_map(hwaddr addr,
3351 hwaddr *plen,
Avi Kivityac1970f2012-10-03 16:22:53 +02003352 int is_write)
3353{
3354 return address_space_map(&address_space_memory, addr, plen, is_write);
3355}
3356
Avi Kivitya8170e52012-10-23 12:30:10 +02003357void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3358 int is_write, hwaddr access_len)
Avi Kivityac1970f2012-10-03 16:22:53 +02003359{
3360 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3361}
3362
Paolo Bonzini0ce265f2016-11-22 11:34:02 +01003363#define ARG1_DECL AddressSpace *as
3364#define ARG1 as
3365#define SUFFIX
3366#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
3367#define IS_DIRECT(mr, is_write) memory_access_is_direct(mr, is_write)
3368#define MAP_RAM(mr, ofs) qemu_map_ram_ptr((mr)->ram_block, ofs)
3369#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len)
3370#define RCU_READ_LOCK(...) rcu_read_lock()
3371#define RCU_READ_UNLOCK(...) rcu_read_unlock()
3372#include "memory_ldst.inc.c"
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003373
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003374int64_t address_space_cache_init(MemoryRegionCache *cache,
3375 AddressSpace *as,
3376 hwaddr addr,
3377 hwaddr len,
3378 bool is_write)
3379{
Paolo Bonzini90c4fe52017-04-03 13:41:28 +02003380 cache->len = len;
3381 cache->as = as;
3382 cache->xlat = addr;
3383 return len;
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003384}
3385
3386void address_space_cache_invalidate(MemoryRegionCache *cache,
3387 hwaddr addr,
3388 hwaddr access_len)
3389{
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003390}
3391
3392void address_space_cache_destroy(MemoryRegionCache *cache)
3393{
Paolo Bonzini90c4fe52017-04-03 13:41:28 +02003394 cache->as = NULL;
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003395}
3396
3397#define ARG1_DECL MemoryRegionCache *cache
3398#define ARG1 cache
3399#define SUFFIX _cached
Paolo Bonzini90c4fe52017-04-03 13:41:28 +02003400#define TRANSLATE(addr, ...) \
3401 address_space_translate(cache->as, cache->xlat + (addr), __VA_ARGS__)
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003402#define IS_DIRECT(mr, is_write) true
Paolo Bonzini90c4fe52017-04-03 13:41:28 +02003403#define MAP_RAM(mr, ofs) qemu_map_ram_ptr((mr)->ram_block, ofs)
3404#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len)
3405#define RCU_READ_LOCK() rcu_read_lock()
3406#define RCU_READ_UNLOCK() rcu_read_unlock()
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003407#include "memory_ldst.inc.c"
3408
aliguori5e2972f2009-03-28 17:51:36 +00003409/* virtual memory access for debug (includes writing to ROM) */
Andreas Färberf17ec442013-06-29 19:40:58 +02003410int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
bellardb448f2f2004-02-25 23:24:04 +00003411 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00003412{
3413 int l;
Avi Kivitya8170e52012-10-23 12:30:10 +02003414 hwaddr phys_addr;
j_mayer9b3c35e2007-04-07 11:21:28 +00003415 target_ulong page;
bellard13eb76e2004-01-24 15:23:36 +00003416
Christian Borntraeger79ca7a12017-03-07 15:19:08 +01003417 cpu_synchronize_state(cpu);
bellard13eb76e2004-01-24 15:23:36 +00003418 while (len > 0) {
Peter Maydell5232e4c2016-01-21 14:15:06 +00003419 int asidx;
3420 MemTxAttrs attrs;
3421
bellard13eb76e2004-01-24 15:23:36 +00003422 page = addr & TARGET_PAGE_MASK;
Peter Maydell5232e4c2016-01-21 14:15:06 +00003423 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3424 asidx = cpu_asidx_from_attrs(cpu, attrs);
bellard13eb76e2004-01-24 15:23:36 +00003425 /* if no physical page mapped, return an error */
3426 if (phys_addr == -1)
3427 return -1;
3428 l = (page + TARGET_PAGE_SIZE) - addr;
3429 if (l > len)
3430 l = len;
aliguori5e2972f2009-03-28 17:51:36 +00003431 phys_addr += (addr & ~TARGET_PAGE_MASK);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003432 if (is_write) {
Peter Maydell5232e4c2016-01-21 14:15:06 +00003433 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3434 phys_addr, buf, l);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003435 } else {
Peter Maydell5232e4c2016-01-21 14:15:06 +00003436 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3437 MEMTXATTRS_UNSPECIFIED,
Peter Maydell5c9eb022015-04-26 16:49:24 +01003438 buf, l, 0);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003439 }
bellard13eb76e2004-01-24 15:23:36 +00003440 len -= l;
3441 buf += l;
3442 addr += l;
3443 }
3444 return 0;
3445}
Dr. David Alan Gilbert038629a2015-11-05 18:10:29 +00003446
3447/*
3448 * Allows code that needs to deal with migration bitmaps etc to still be built
3449 * target independent.
3450 */
Juan Quintela20afaed2017-03-21 09:09:14 +01003451size_t qemu_target_page_size(void)
Dr. David Alan Gilbert038629a2015-11-05 18:10:29 +00003452{
Juan Quintela20afaed2017-03-21 09:09:14 +01003453 return TARGET_PAGE_SIZE;
Dr. David Alan Gilbert038629a2015-11-05 18:10:29 +00003454}
3455
Juan Quintela46d702b2017-04-24 21:03:48 +02003456int qemu_target_page_bits(void)
3457{
3458 return TARGET_PAGE_BITS;
3459}
3460
3461int qemu_target_page_bits_min(void)
3462{
3463 return TARGET_PAGE_BITS_MIN;
3464}
Paul Brooka68fe892010-03-01 00:08:59 +00003465#endif
bellard13eb76e2004-01-24 15:23:36 +00003466
Blue Swirl8e4a4242013-01-06 18:30:17 +00003467/*
3468 * A helper function for the _utterly broken_ virtio device model to find out if
3469 * it's running on a big endian machine. Don't do this at home kids!
3470 */
Greg Kurz98ed8ec2014-06-24 19:26:29 +02003471bool target_words_bigendian(void);
3472bool target_words_bigendian(void)
Blue Swirl8e4a4242013-01-06 18:30:17 +00003473{
3474#if defined(TARGET_WORDS_BIGENDIAN)
3475 return true;
3476#else
3477 return false;
3478#endif
3479}
3480
Wen Congyang76f35532012-05-07 12:04:18 +08003481#ifndef CONFIG_USER_ONLY
Avi Kivitya8170e52012-10-23 12:30:10 +02003482bool cpu_physical_memory_is_io(hwaddr phys_addr)
Wen Congyang76f35532012-05-07 12:04:18 +08003483{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003484 MemoryRegion*mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003485 hwaddr l = 1;
Paolo Bonzini41063e12015-03-18 14:21:43 +01003486 bool res;
Wen Congyang76f35532012-05-07 12:04:18 +08003487
Paolo Bonzini41063e12015-03-18 14:21:43 +01003488 rcu_read_lock();
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003489 mr = address_space_translate(&address_space_memory,
3490 phys_addr, &phys_addr, &l, false);
Wen Congyang76f35532012-05-07 12:04:18 +08003491
Paolo Bonzini41063e12015-03-18 14:21:43 +01003492 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3493 rcu_read_unlock();
3494 return res;
Wen Congyang76f35532012-05-07 12:04:18 +08003495}
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003496
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003497int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003498{
3499 RAMBlock *block;
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003500 int ret = 0;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003501
Mike Day0dc3f442013-09-05 14:41:35 -04003502 rcu_read_lock();
Peter Xu99e15582017-05-12 12:17:39 +08003503 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003504 ret = func(block->idstr, block->host, block->offset,
3505 block->used_length, opaque);
3506 if (ret) {
3507 break;
3508 }
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003509 }
Mike Day0dc3f442013-09-05 14:41:35 -04003510 rcu_read_unlock();
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003511 return ret;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003512}
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003513
3514/*
3515 * Unmap pages of memory from start to start+length such that
3516 * they a) read as 0, b) Trigger whatever fault mechanism
3517 * the OS provides for postcopy.
3518 * The pages must be unmapped by the end of the function.
3519 * Returns: 0 on success, none-0 on failure
3520 *
3521 */
3522int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3523{
3524 int ret = -1;
3525
3526 uint8_t *host_startaddr = rb->host + start;
3527
3528 if ((uintptr_t)host_startaddr & (rb->page_size - 1)) {
3529 error_report("ram_block_discard_range: Unaligned start address: %p",
3530 host_startaddr);
3531 goto err;
3532 }
3533
3534 if ((start + length) <= rb->used_length) {
3535 uint8_t *host_endaddr = host_startaddr + length;
3536 if ((uintptr_t)host_endaddr & (rb->page_size - 1)) {
3537 error_report("ram_block_discard_range: Unaligned end address: %p",
3538 host_endaddr);
3539 goto err;
3540 }
3541
3542 errno = ENOTSUP; /* If we are missing MADVISE etc */
3543
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +00003544 if (rb->page_size == qemu_host_page_size) {
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003545#if defined(CONFIG_MADVISE)
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +00003546 /* Note: We need the madvise MADV_DONTNEED behaviour of definitely
3547 * freeing the page.
3548 */
3549 ret = madvise(host_startaddr, length, MADV_DONTNEED);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003550#endif
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +00003551 } else {
3552 /* Huge page case - unfortunately it can't do DONTNEED, but
3553 * it can do the equivalent by FALLOC_FL_PUNCH_HOLE in the
3554 * huge page file.
3555 */
3556#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3557 ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3558 start, length);
3559#endif
3560 }
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003561 if (ret) {
3562 ret = -errno;
3563 error_report("ram_block_discard_range: Failed to discard range "
3564 "%s:%" PRIx64 " +%zx (%d)",
3565 rb->idstr, start, length, ret);
3566 }
3567 } else {
3568 error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
3569 "/%zx/" RAM_ADDR_FMT")",
3570 rb->idstr, start, length, rb->used_length);
3571 }
3572
3573err:
3574 return ret;
3575}
3576
Peter Maydellec3f8c92013-06-27 20:53:38 +01003577#endif