blob: 751fd8967b5dfcc1e934afadd8485b343b95a863 [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
bellardfd6ce8f2003-05-14 19:00:11 +00002 * virtual page mapping and translated block handling
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
bellard67b915a2004-03-31 23:37:16 +000019#include "config.h"
bellardd5a8f072004-09-29 21:15:28 +000020#ifdef _WIN32
21#include <windows.h>
22#else
bellarda98d49b2004-11-14 16:22:05 +000023#include <sys/types.h>
bellardd5a8f072004-09-29 21:15:28 +000024#include <sys/mman.h>
25#endif
bellard54936002003-05-13 00:25:15 +000026
Stefan Weil055403b2010-10-22 23:03:32 +020027#include "qemu-common.h"
bellard6180a182003-09-30 21:04:53 +000028#include "cpu.h"
bellardb67d9a52008-05-23 09:57:34 +000029#include "tcg.h"
pbrookb3c77242008-06-30 16:31:04 +000030#include "hw/hw.h"
Alex Williamsoncc9e98c2010-06-25 11:09:43 -060031#include "hw/qdev.h"
aliguori74576192008-10-06 14:02:03 +000032#include "osdep.h"
aliguori7ba1e612008-11-05 16:04:33 +000033#include "kvm.h"
Jun Nakajima432d2682010-08-31 16:41:25 +010034#include "hw/xen.h"
Blue Swirl29e922b2010-03-29 19:24:00 +000035#include "qemu-timer.h"
Avi Kivity62152b82011-07-26 14:26:14 +030036#include "memory.h"
37#include "exec-memory.h"
pbrook53a59602006-03-25 19:31:22 +000038#if defined(CONFIG_USER_ONLY)
39#include <qemu.h>
Juergen Lockf01576f2010-03-25 22:32:16 +010040#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41#include <sys/param.h>
42#if __FreeBSD_version >= 700104
43#define HAVE_KINFO_GETVMMAP
44#define sigqueue sigqueue_freebsd /* avoid redefinition */
45#include <sys/time.h>
46#include <sys/proc.h>
47#include <machine/profile.h>
48#define _KERNEL
49#include <sys/user.h>
50#undef _KERNEL
51#undef sigqueue
52#include <libutil.h>
53#endif
54#endif
Jun Nakajima432d2682010-08-31 16:41:25 +010055#else /* !CONFIG_USER_ONLY */
56#include "xen-mapcache.h"
Stefano Stabellini6506e4f2011-05-19 18:35:44 +010057#include "trace.h"
pbrook53a59602006-03-25 19:31:22 +000058#endif
bellard54936002003-05-13 00:25:15 +000059
bellardfd6ce8f2003-05-14 19:00:11 +000060//#define DEBUG_TB_INVALIDATE
bellard66e85a22003-06-24 13:28:12 +000061//#define DEBUG_FLUSH
bellard9fa3e852004-01-04 18:06:42 +000062//#define DEBUG_TLB
pbrook67d3b952006-12-18 05:03:52 +000063//#define DEBUG_UNASSIGNED
bellardfd6ce8f2003-05-14 19:00:11 +000064
65/* make various TB consistency checks */
ths5fafdf22007-09-16 21:08:06 +000066//#define DEBUG_TB_CHECK
67//#define DEBUG_TLB_CHECK
bellardfd6ce8f2003-05-14 19:00:11 +000068
ths1196be32007-03-17 15:17:58 +000069//#define DEBUG_IOPORT
blueswir1db7b5422007-05-26 17:36:03 +000070//#define DEBUG_SUBPAGE
ths1196be32007-03-17 15:17:58 +000071
pbrook99773bd2006-04-16 15:14:59 +000072#if !defined(CONFIG_USER_ONLY)
73/* TB consistency checks only implemented for usermode emulation. */
74#undef DEBUG_TB_CHECK
75#endif
76
bellard9fa3e852004-01-04 18:06:42 +000077#define SMC_BITMAP_USE_THRESHOLD 10
78
blueswir1bdaf78e2008-10-04 07:24:27 +000079static TranslationBlock *tbs;
Stefan Weil24ab68a2010-07-19 18:23:17 +020080static int code_gen_max_blocks;
bellard9fa3e852004-01-04 18:06:42 +000081TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
blueswir1bdaf78e2008-10-04 07:24:27 +000082static int nb_tbs;
bellardeb51d102003-05-14 21:51:13 +000083/* any access to the tbs or the page table must use this lock */
Anthony Liguoric227f092009-10-01 16:12:16 -050084spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
bellardfd6ce8f2003-05-14 19:00:11 +000085
blueswir1141ac462008-07-26 15:05:57 +000086#if defined(__arm__) || defined(__sparc_v9__)
87/* The prologue must be reachable with a direct jump. ARM and Sparc64
88 have limited branch ranges (possibly also PPC) so place it in a
blueswir1d03d8602008-07-10 17:21:31 +000089 section close to code segment. */
90#define code_gen_section \
91 __attribute__((__section__(".gen_code"))) \
92 __attribute__((aligned (32)))
Stefan Weilf8e2af12009-06-18 23:04:48 +020093#elif defined(_WIN32)
94/* Maximum alignment for Win32 is 16. */
95#define code_gen_section \
96 __attribute__((aligned (16)))
blueswir1d03d8602008-07-10 17:21:31 +000097#else
98#define code_gen_section \
99 __attribute__((aligned (32)))
100#endif
101
102uint8_t code_gen_prologue[1024] code_gen_section;
blueswir1bdaf78e2008-10-04 07:24:27 +0000103static uint8_t *code_gen_buffer;
104static unsigned long code_gen_buffer_size;
bellard26a5f132008-05-28 12:30:31 +0000105/* threshold to flush the translated code buffer */
blueswir1bdaf78e2008-10-04 07:24:27 +0000106static unsigned long code_gen_buffer_max_size;
Stefan Weil24ab68a2010-07-19 18:23:17 +0200107static uint8_t *code_gen_ptr;
bellardfd6ce8f2003-05-14 19:00:11 +0000108
pbrooke2eef172008-06-08 01:09:01 +0000109#if !defined(CONFIG_USER_ONLY)
bellard9fa3e852004-01-04 18:06:42 +0000110int phys_ram_fd;
aliguori74576192008-10-06 14:02:03 +0000111static int in_migration;
pbrook94a6b542009-04-11 17:15:54 +0000112
Alex Williamsonf471a172010-06-11 11:11:42 -0600113RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
Avi Kivity62152b82011-07-26 14:26:14 +0300114
115static MemoryRegion *system_memory;
116
pbrooke2eef172008-06-08 01:09:01 +0000117#endif
bellard9fa3e852004-01-04 18:06:42 +0000118
bellard6a00d602005-11-21 23:25:50 +0000119CPUState *first_cpu;
120/* current CPU in the current thread. It is only valid inside
121 cpu_exec() */
ths5fafdf22007-09-16 21:08:06 +0000122CPUState *cpu_single_env;
pbrook2e70f6e2008-06-29 01:03:05 +0000123/* 0 = Do not count executed instructions.
thsbf20dc02008-06-30 17:22:19 +0000124 1 = Precise instruction counting.
pbrook2e70f6e2008-06-29 01:03:05 +0000125 2 = Adaptive rate instruction counting. */
126int use_icount = 0;
127/* Current instruction counter. While executing translated code this may
128 include some instructions that have not yet been executed. */
129int64_t qemu_icount;
bellard6a00d602005-11-21 23:25:50 +0000130
bellard54936002003-05-13 00:25:15 +0000131typedef struct PageDesc {
bellard92e873b2004-05-21 14:52:29 +0000132 /* list of TBs intersecting this ram page */
bellardfd6ce8f2003-05-14 19:00:11 +0000133 TranslationBlock *first_tb;
bellard9fa3e852004-01-04 18:06:42 +0000134 /* in order to optimize self modifying code, we count the number
135 of lookups we do to a given page to use a bitmap */
136 unsigned int code_write_count;
137 uint8_t *code_bitmap;
138#if defined(CONFIG_USER_ONLY)
139 unsigned long flags;
140#endif
bellard54936002003-05-13 00:25:15 +0000141} PageDesc;
142
Paul Brook41c1b1c2010-03-12 16:54:58 +0000143/* In system mode we want L1_MAP to be based on ram offsets,
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800144 while in user mode we want it to be based on virtual addresses. */
145#if !defined(CONFIG_USER_ONLY)
Paul Brook41c1b1c2010-03-12 16:54:58 +0000146#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
147# define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
148#else
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800149# define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
Paul Brook41c1b1c2010-03-12 16:54:58 +0000150#endif
j_mayerbedb69e2007-04-05 20:08:21 +0000151#else
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800152# define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
j_mayerbedb69e2007-04-05 20:08:21 +0000153#endif
bellard54936002003-05-13 00:25:15 +0000154
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800155/* Size of the L2 (and L3, etc) page tables. */
156#define L2_BITS 10
bellard54936002003-05-13 00:25:15 +0000157#define L2_SIZE (1 << L2_BITS)
158
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800159/* The bits remaining after N lower levels of page tables. */
160#define P_L1_BITS_REM \
161 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
162#define V_L1_BITS_REM \
163 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
164
165/* Size of the L1 page table. Avoid silly small sizes. */
166#if P_L1_BITS_REM < 4
167#define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
168#else
169#define P_L1_BITS P_L1_BITS_REM
170#endif
171
172#if V_L1_BITS_REM < 4
173#define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
174#else
175#define V_L1_BITS V_L1_BITS_REM
176#endif
177
178#define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
179#define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
180
181#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
182#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
183
bellard83fb7ad2004-07-05 21:25:26 +0000184unsigned long qemu_real_host_page_size;
185unsigned long qemu_host_page_bits;
186unsigned long qemu_host_page_size;
187unsigned long qemu_host_page_mask;
bellard54936002003-05-13 00:25:15 +0000188
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800189/* This is a multi-level map on the virtual address space.
190 The bottom level has pointers to PageDesc. */
191static void *l1_map[V_L1_SIZE];
bellard54936002003-05-13 00:25:15 +0000192
pbrooke2eef172008-06-08 01:09:01 +0000193#if !defined(CONFIG_USER_ONLY)
Paul Brook41c1b1c2010-03-12 16:54:58 +0000194typedef struct PhysPageDesc {
195 /* offset in host memory of the page + io_index in the low bits */
196 ram_addr_t phys_offset;
197 ram_addr_t region_offset;
198} PhysPageDesc;
199
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800200/* This is a multi-level map on the physical address space.
201 The bottom level has pointers to PhysPageDesc. */
202static void *l1_phys_map[P_L1_SIZE];
Paul Brook6d9a1302010-02-28 23:55:53 +0000203
pbrooke2eef172008-06-08 01:09:01 +0000204static void io_mem_init(void);
Avi Kivity62152b82011-07-26 14:26:14 +0300205static void memory_map_init(void);
pbrooke2eef172008-06-08 01:09:01 +0000206
bellard33417e72003-08-10 21:47:01 +0000207/* io memory support */
bellard33417e72003-08-10 21:47:01 +0000208CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
209CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
bellarda4193c82004-06-03 14:01:43 +0000210void *io_mem_opaque[IO_MEM_NB_ENTRIES];
blueswir1511d2b12009-03-07 15:32:56 +0000211static char io_mem_used[IO_MEM_NB_ENTRIES];
pbrook6658ffb2007-03-16 23:58:11 +0000212static int io_mem_watch;
213#endif
bellard33417e72003-08-10 21:47:01 +0000214
bellard34865132003-10-05 14:28:56 +0000215/* log support */
Juha Riihimäki1e8b27c2009-12-03 15:56:02 +0200216#ifdef WIN32
217static const char *logfilename = "qemu.log";
218#else
blueswir1d9b630f2008-10-05 09:57:08 +0000219static const char *logfilename = "/tmp/qemu.log";
Juha Riihimäki1e8b27c2009-12-03 15:56:02 +0200220#endif
bellard34865132003-10-05 14:28:56 +0000221FILE *logfile;
222int loglevel;
pbrooke735b912007-06-30 13:53:24 +0000223static int log_append = 0;
bellard34865132003-10-05 14:28:56 +0000224
bellarde3db7222005-01-26 22:00:47 +0000225/* statistics */
Paul Brookb3755a92010-03-12 16:54:58 +0000226#if !defined(CONFIG_USER_ONLY)
bellarde3db7222005-01-26 22:00:47 +0000227static int tlb_flush_count;
Paul Brookb3755a92010-03-12 16:54:58 +0000228#endif
bellarde3db7222005-01-26 22:00:47 +0000229static int tb_flush_count;
230static int tb_phys_invalidate_count;
231
bellard7cb69ca2008-05-10 10:55:51 +0000232#ifdef _WIN32
233static void map_exec(void *addr, long size)
234{
235 DWORD old_protect;
236 VirtualProtect(addr, size,
237 PAGE_EXECUTE_READWRITE, &old_protect);
238
239}
240#else
241static void map_exec(void *addr, long size)
242{
bellard43694152008-05-29 09:35:57 +0000243 unsigned long start, end, page_size;
bellard7cb69ca2008-05-10 10:55:51 +0000244
bellard43694152008-05-29 09:35:57 +0000245 page_size = getpagesize();
bellard7cb69ca2008-05-10 10:55:51 +0000246 start = (unsigned long)addr;
bellard43694152008-05-29 09:35:57 +0000247 start &= ~(page_size - 1);
bellard7cb69ca2008-05-10 10:55:51 +0000248
249 end = (unsigned long)addr + size;
bellard43694152008-05-29 09:35:57 +0000250 end += page_size - 1;
251 end &= ~(page_size - 1);
bellard7cb69ca2008-05-10 10:55:51 +0000252
253 mprotect((void *)start, end - start,
254 PROT_READ | PROT_WRITE | PROT_EXEC);
255}
256#endif
257
bellardb346ff42003-06-15 20:05:50 +0000258static void page_init(void)
bellard54936002003-05-13 00:25:15 +0000259{
bellard83fb7ad2004-07-05 21:25:26 +0000260 /* NOTE: we can always suppose that qemu_host_page_size >=
bellard54936002003-05-13 00:25:15 +0000261 TARGET_PAGE_SIZE */
aliguoric2b48b62008-11-11 22:06:42 +0000262#ifdef _WIN32
263 {
264 SYSTEM_INFO system_info;
265
266 GetSystemInfo(&system_info);
267 qemu_real_host_page_size = system_info.dwPageSize;
268 }
269#else
270 qemu_real_host_page_size = getpagesize();
271#endif
bellard83fb7ad2004-07-05 21:25:26 +0000272 if (qemu_host_page_size == 0)
273 qemu_host_page_size = qemu_real_host_page_size;
274 if (qemu_host_page_size < TARGET_PAGE_SIZE)
275 qemu_host_page_size = TARGET_PAGE_SIZE;
276 qemu_host_page_bits = 0;
277 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
278 qemu_host_page_bits++;
279 qemu_host_page_mask = ~(qemu_host_page_size - 1);
balrog50a95692007-12-12 01:16:23 +0000280
Paul Brook2e9a5712010-05-05 16:32:59 +0100281#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
balrog50a95692007-12-12 01:16:23 +0000282 {
Juergen Lockf01576f2010-03-25 22:32:16 +0100283#ifdef HAVE_KINFO_GETVMMAP
284 struct kinfo_vmentry *freep;
285 int i, cnt;
286
287 freep = kinfo_getvmmap(getpid(), &cnt);
288 if (freep) {
289 mmap_lock();
290 for (i = 0; i < cnt; i++) {
291 unsigned long startaddr, endaddr;
292
293 startaddr = freep[i].kve_start;
294 endaddr = freep[i].kve_end;
295 if (h2g_valid(startaddr)) {
296 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
297
298 if (h2g_valid(endaddr)) {
299 endaddr = h2g(endaddr);
Aurelien Jarnofd436902010-04-10 17:20:36 +0200300 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
Juergen Lockf01576f2010-03-25 22:32:16 +0100301 } else {
302#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
303 endaddr = ~0ul;
Aurelien Jarnofd436902010-04-10 17:20:36 +0200304 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
Juergen Lockf01576f2010-03-25 22:32:16 +0100305#endif
306 }
307 }
308 }
309 free(freep);
310 mmap_unlock();
311 }
312#else
balrog50a95692007-12-12 01:16:23 +0000313 FILE *f;
balrog50a95692007-12-12 01:16:23 +0000314
pbrook07765902008-05-31 16:33:53 +0000315 last_brk = (unsigned long)sbrk(0);
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800316
Aurelien Jarnofd436902010-04-10 17:20:36 +0200317 f = fopen("/compat/linux/proc/self/maps", "r");
balrog50a95692007-12-12 01:16:23 +0000318 if (f) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800319 mmap_lock();
320
balrog50a95692007-12-12 01:16:23 +0000321 do {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800322 unsigned long startaddr, endaddr;
323 int n;
324
325 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
326
327 if (n == 2 && h2g_valid(startaddr)) {
328 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
329
330 if (h2g_valid(endaddr)) {
331 endaddr = h2g(endaddr);
332 } else {
333 endaddr = ~0ul;
334 }
335 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
balrog50a95692007-12-12 01:16:23 +0000336 }
337 } while (!feof(f));
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800338
balrog50a95692007-12-12 01:16:23 +0000339 fclose(f);
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800340 mmap_unlock();
balrog50a95692007-12-12 01:16:23 +0000341 }
Juergen Lockf01576f2010-03-25 22:32:16 +0100342#endif
balrog50a95692007-12-12 01:16:23 +0000343 }
344#endif
bellard54936002003-05-13 00:25:15 +0000345}
346
Paul Brook41c1b1c2010-03-12 16:54:58 +0000347static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
bellard54936002003-05-13 00:25:15 +0000348{
Paul Brook41c1b1c2010-03-12 16:54:58 +0000349 PageDesc *pd;
350 void **lp;
351 int i;
352
pbrook17e23772008-06-09 13:47:45 +0000353#if defined(CONFIG_USER_ONLY)
Paul Brook2e9a5712010-05-05 16:32:59 +0100354 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800355# define ALLOC(P, SIZE) \
356 do { \
357 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
358 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800359 } while (0)
pbrook17e23772008-06-09 13:47:45 +0000360#else
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800361# define ALLOC(P, SIZE) \
362 do { P = qemu_mallocz(SIZE); } while (0)
pbrook17e23772008-06-09 13:47:45 +0000363#endif
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800364
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800365 /* Level 1. Always allocated. */
366 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
367
368 /* Level 2..N-1. */
369 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
370 void **p = *lp;
371
372 if (p == NULL) {
373 if (!alloc) {
374 return NULL;
375 }
376 ALLOC(p, sizeof(void *) * L2_SIZE);
377 *lp = p;
378 }
379
380 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
bellard54936002003-05-13 00:25:15 +0000381 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800382
383 pd = *lp;
384 if (pd == NULL) {
385 if (!alloc) {
386 return NULL;
387 }
388 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
389 *lp = pd;
390 }
391
392#undef ALLOC
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800393
394 return pd + (index & (L2_SIZE - 1));
bellard54936002003-05-13 00:25:15 +0000395}
396
Paul Brook41c1b1c2010-03-12 16:54:58 +0000397static inline PageDesc *page_find(tb_page_addr_t index)
bellard54936002003-05-13 00:25:15 +0000398{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800399 return page_find_alloc(index, 0);
bellard54936002003-05-13 00:25:15 +0000400}
401
Paul Brook6d9a1302010-02-28 23:55:53 +0000402#if !defined(CONFIG_USER_ONLY)
Anthony Liguoric227f092009-10-01 16:12:16 -0500403static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
bellard92e873b2004-05-21 14:52:29 +0000404{
pbrooke3f4e2a2006-04-08 20:02:06 +0000405 PhysPageDesc *pd;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800406 void **lp;
407 int i;
bellard92e873b2004-05-21 14:52:29 +0000408
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800409 /* Level 1. Always allocated. */
410 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
bellard108c49b2005-07-24 12:55:09 +0000411
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800412 /* Level 2..N-1. */
413 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
414 void **p = *lp;
415 if (p == NULL) {
416 if (!alloc) {
417 return NULL;
418 }
419 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
420 }
421 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
bellard108c49b2005-07-24 12:55:09 +0000422 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800423
pbrooke3f4e2a2006-04-08 20:02:06 +0000424 pd = *lp;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800425 if (pd == NULL) {
pbrooke3f4e2a2006-04-08 20:02:06 +0000426 int i;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800427
428 if (!alloc) {
bellard108c49b2005-07-24 12:55:09 +0000429 return NULL;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800430 }
431
432 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
433
pbrook67c4d232009-02-23 13:16:07 +0000434 for (i = 0; i < L2_SIZE; i++) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800435 pd[i].phys_offset = IO_MEM_UNASSIGNED;
436 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
pbrook67c4d232009-02-23 13:16:07 +0000437 }
bellard92e873b2004-05-21 14:52:29 +0000438 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800439
440 return pd + (index & (L2_SIZE - 1));
bellard92e873b2004-05-21 14:52:29 +0000441}
442
Anthony Liguoric227f092009-10-01 16:12:16 -0500443static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
bellard92e873b2004-05-21 14:52:29 +0000444{
bellard108c49b2005-07-24 12:55:09 +0000445 return phys_page_find_alloc(index, 0);
bellard92e873b2004-05-21 14:52:29 +0000446}
447
Anthony Liguoric227f092009-10-01 16:12:16 -0500448static void tlb_protect_code(ram_addr_t ram_addr);
449static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
bellard3a7d9292005-08-21 09:26:42 +0000450 target_ulong vaddr);
pbrookc8a706f2008-06-02 16:16:42 +0000451#define mmap_lock() do { } while(0)
452#define mmap_unlock() do { } while(0)
bellard9fa3e852004-01-04 18:06:42 +0000453#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000454
bellard43694152008-05-29 09:35:57 +0000455#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
456
457#if defined(CONFIG_USER_ONLY)
Stuart Bradyccbb4d42009-05-03 12:15:06 +0100458/* Currently it is not recommended to allocate big chunks of data in
bellard43694152008-05-29 09:35:57 +0000459 user mode. It will change when a dedicated libc will be used */
460#define USE_STATIC_CODE_GEN_BUFFER
461#endif
462
463#ifdef USE_STATIC_CODE_GEN_BUFFER
Aurelien Jarnoebf50fb2010-03-29 02:12:51 +0200464static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
465 __attribute__((aligned (CODE_GEN_ALIGN)));
bellard43694152008-05-29 09:35:57 +0000466#endif
467
blueswir18fcd3692008-08-17 20:26:25 +0000468static void code_gen_alloc(unsigned long tb_size)
bellard26a5f132008-05-28 12:30:31 +0000469{
bellard43694152008-05-29 09:35:57 +0000470#ifdef USE_STATIC_CODE_GEN_BUFFER
471 code_gen_buffer = static_code_gen_buffer;
472 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
473 map_exec(code_gen_buffer, code_gen_buffer_size);
474#else
bellard26a5f132008-05-28 12:30:31 +0000475 code_gen_buffer_size = tb_size;
476 if (code_gen_buffer_size == 0) {
bellard43694152008-05-29 09:35:57 +0000477#if defined(CONFIG_USER_ONLY)
478 /* in user mode, phys_ram_size is not meaningful */
479 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
480#else
Stuart Bradyccbb4d42009-05-03 12:15:06 +0100481 /* XXX: needs adjustments */
pbrook94a6b542009-04-11 17:15:54 +0000482 code_gen_buffer_size = (unsigned long)(ram_size / 4);
bellard43694152008-05-29 09:35:57 +0000483#endif
bellard26a5f132008-05-28 12:30:31 +0000484 }
485 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
486 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
487 /* The code gen buffer location may have constraints depending on
488 the host cpu and OS */
489#if defined(__linux__)
490 {
491 int flags;
blueswir1141ac462008-07-26 15:05:57 +0000492 void *start = NULL;
493
bellard26a5f132008-05-28 12:30:31 +0000494 flags = MAP_PRIVATE | MAP_ANONYMOUS;
495#if defined(__x86_64__)
496 flags |= MAP_32BIT;
497 /* Cannot map more than that */
498 if (code_gen_buffer_size > (800 * 1024 * 1024))
499 code_gen_buffer_size = (800 * 1024 * 1024);
blueswir1141ac462008-07-26 15:05:57 +0000500#elif defined(__sparc_v9__)
501 // Map the buffer below 2G, so we can use direct calls and branches
502 flags |= MAP_FIXED;
503 start = (void *) 0x60000000UL;
504 if (code_gen_buffer_size > (512 * 1024 * 1024))
505 code_gen_buffer_size = (512 * 1024 * 1024);
balrog1cb06612008-12-01 02:10:17 +0000506#elif defined(__arm__)
balrog63d41242008-12-01 02:19:41 +0000507 /* Map the buffer below 32M, so we can use direct calls and branches */
balrog1cb06612008-12-01 02:10:17 +0000508 flags |= MAP_FIXED;
509 start = (void *) 0x01000000UL;
510 if (code_gen_buffer_size > 16 * 1024 * 1024)
511 code_gen_buffer_size = 16 * 1024 * 1024;
Richard Hendersoneba0b892010-06-04 12:14:14 -0700512#elif defined(__s390x__)
513 /* Map the buffer so that we can use direct calls and branches. */
514 /* We have a +- 4GB range on the branches; leave some slop. */
515 if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
516 code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
517 }
518 start = (void *)0x90000000UL;
bellard26a5f132008-05-28 12:30:31 +0000519#endif
blueswir1141ac462008-07-26 15:05:57 +0000520 code_gen_buffer = mmap(start, code_gen_buffer_size,
521 PROT_WRITE | PROT_READ | PROT_EXEC,
bellard26a5f132008-05-28 12:30:31 +0000522 flags, -1, 0);
523 if (code_gen_buffer == MAP_FAILED) {
524 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
525 exit(1);
526 }
527 }
Bradcbb608a2010-12-20 21:25:40 -0500528#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
529 || defined(__DragonFly__) || defined(__OpenBSD__)
aliguori06e67a82008-09-27 15:32:41 +0000530 {
531 int flags;
532 void *addr = NULL;
533 flags = MAP_PRIVATE | MAP_ANONYMOUS;
534#if defined(__x86_64__)
535 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
536 * 0x40000000 is free */
537 flags |= MAP_FIXED;
538 addr = (void *)0x40000000;
539 /* Cannot map more than that */
540 if (code_gen_buffer_size > (800 * 1024 * 1024))
541 code_gen_buffer_size = (800 * 1024 * 1024);
Blue Swirl4cd31ad2011-01-16 08:32:27 +0000542#elif defined(__sparc_v9__)
543 // Map the buffer below 2G, so we can use direct calls and branches
544 flags |= MAP_FIXED;
545 addr = (void *) 0x60000000UL;
546 if (code_gen_buffer_size > (512 * 1024 * 1024)) {
547 code_gen_buffer_size = (512 * 1024 * 1024);
548 }
aliguori06e67a82008-09-27 15:32:41 +0000549#endif
550 code_gen_buffer = mmap(addr, code_gen_buffer_size,
551 PROT_WRITE | PROT_READ | PROT_EXEC,
552 flags, -1, 0);
553 if (code_gen_buffer == MAP_FAILED) {
554 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
555 exit(1);
556 }
557 }
bellard26a5f132008-05-28 12:30:31 +0000558#else
559 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
bellard26a5f132008-05-28 12:30:31 +0000560 map_exec(code_gen_buffer, code_gen_buffer_size);
561#endif
bellard43694152008-05-29 09:35:57 +0000562#endif /* !USE_STATIC_CODE_GEN_BUFFER */
bellard26a5f132008-05-28 12:30:31 +0000563 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
Peter Maydella884da82011-06-22 11:58:25 +0100564 code_gen_buffer_max_size = code_gen_buffer_size -
565 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
bellard26a5f132008-05-28 12:30:31 +0000566 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
567 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
568}
569
570/* Must be called before using the QEMU cpus. 'tb_size' is the size
571 (in bytes) allocated to the translation buffer. Zero means default
572 size. */
573void cpu_exec_init_all(unsigned long tb_size)
574{
bellard26a5f132008-05-28 12:30:31 +0000575 cpu_gen_init();
576 code_gen_alloc(tb_size);
577 code_gen_ptr = code_gen_buffer;
bellard43694152008-05-29 09:35:57 +0000578 page_init();
pbrooke2eef172008-06-08 01:09:01 +0000579#if !defined(CONFIG_USER_ONLY)
Avi Kivity62152b82011-07-26 14:26:14 +0300580 memory_map_init();
bellard26a5f132008-05-28 12:30:31 +0000581 io_mem_init();
pbrooke2eef172008-06-08 01:09:01 +0000582#endif
Richard Henderson9002ec72010-05-06 08:50:41 -0700583#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
584 /* There's no guest base to take into account, so go ahead and
585 initialize the prologue now. */
586 tcg_prologue_init(&tcg_ctx);
587#endif
bellard26a5f132008-05-28 12:30:31 +0000588}
589
pbrook9656f322008-07-01 20:01:19 +0000590#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
591
Juan Quintelae59fb372009-09-29 22:48:21 +0200592static int cpu_common_post_load(void *opaque, int version_id)
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200593{
594 CPUState *env = opaque;
595
aurel323098dba2009-03-07 21:28:24 +0000596 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
597 version_id is increased. */
598 env->interrupt_request &= ~0x01;
pbrook9656f322008-07-01 20:01:19 +0000599 tlb_flush(env, 1);
600
601 return 0;
602}
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200603
604static const VMStateDescription vmstate_cpu_common = {
605 .name = "cpu_common",
606 .version_id = 1,
607 .minimum_version_id = 1,
608 .minimum_version_id_old = 1,
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200609 .post_load = cpu_common_post_load,
610 .fields = (VMStateField []) {
611 VMSTATE_UINT32(halted, CPUState),
612 VMSTATE_UINT32(interrupt_request, CPUState),
613 VMSTATE_END_OF_LIST()
614 }
615};
pbrook9656f322008-07-01 20:01:19 +0000616#endif
617
Glauber Costa950f1472009-06-09 12:15:18 -0400618CPUState *qemu_get_cpu(int cpu)
619{
620 CPUState *env = first_cpu;
621
622 while (env) {
623 if (env->cpu_index == cpu)
624 break;
625 env = env->next_cpu;
626 }
627
628 return env;
629}
630
bellard6a00d602005-11-21 23:25:50 +0000631void cpu_exec_init(CPUState *env)
bellardfd6ce8f2003-05-14 19:00:11 +0000632{
bellard6a00d602005-11-21 23:25:50 +0000633 CPUState **penv;
634 int cpu_index;
635
pbrookc2764712009-03-07 15:24:59 +0000636#if defined(CONFIG_USER_ONLY)
637 cpu_list_lock();
638#endif
bellard6a00d602005-11-21 23:25:50 +0000639 env->next_cpu = NULL;
640 penv = &first_cpu;
641 cpu_index = 0;
642 while (*penv != NULL) {
Nathan Froyd1e9fa732009-06-03 11:33:08 -0700643 penv = &(*penv)->next_cpu;
bellard6a00d602005-11-21 23:25:50 +0000644 cpu_index++;
645 }
646 env->cpu_index = cpu_index;
aliguori268a3622009-04-21 22:30:27 +0000647 env->numa_node = 0;
Blue Swirl72cf2d42009-09-12 07:36:22 +0000648 QTAILQ_INIT(&env->breakpoints);
649 QTAILQ_INIT(&env->watchpoints);
Jan Kiszkadc7a09c2011-03-15 12:26:31 +0100650#ifndef CONFIG_USER_ONLY
651 env->thread_id = qemu_get_thread_id();
652#endif
bellard6a00d602005-11-21 23:25:50 +0000653 *penv = env;
pbrookc2764712009-03-07 15:24:59 +0000654#if defined(CONFIG_USER_ONLY)
655 cpu_list_unlock();
656#endif
pbrookb3c77242008-06-30 16:31:04 +0000657#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
Alex Williamson0be71e32010-06-25 11:09:07 -0600658 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
659 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
pbrookb3c77242008-06-30 16:31:04 +0000660 cpu_save, cpu_load, env);
661#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000662}
663
Tristan Gingoldd1a1eb72011-02-10 10:04:57 +0100664/* Allocate a new translation block. Flush the translation buffer if
665 too many translation blocks or too much generated code. */
666static TranslationBlock *tb_alloc(target_ulong pc)
667{
668 TranslationBlock *tb;
669
670 if (nb_tbs >= code_gen_max_blocks ||
671 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
672 return NULL;
673 tb = &tbs[nb_tbs++];
674 tb->pc = pc;
675 tb->cflags = 0;
676 return tb;
677}
678
679void tb_free(TranslationBlock *tb)
680{
681 /* In practice this is mostly used for single use temporary TB
682 Ignore the hard cases and just back up if this TB happens to
683 be the last one generated. */
684 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
685 code_gen_ptr = tb->tc_ptr;
686 nb_tbs--;
687 }
688}
689
bellard9fa3e852004-01-04 18:06:42 +0000690static inline void invalidate_page_bitmap(PageDesc *p)
691{
692 if (p->code_bitmap) {
bellard59817cc2004-02-16 22:01:13 +0000693 qemu_free(p->code_bitmap);
bellard9fa3e852004-01-04 18:06:42 +0000694 p->code_bitmap = NULL;
695 }
696 p->code_write_count = 0;
697}
698
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800699/* Set to NULL all the 'first_tb' fields in all PageDescs. */
700
701static void page_flush_tb_1 (int level, void **lp)
702{
703 int i;
704
705 if (*lp == NULL) {
706 return;
707 }
708 if (level == 0) {
709 PageDesc *pd = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +0000710 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800711 pd[i].first_tb = NULL;
712 invalidate_page_bitmap(pd + i);
713 }
714 } else {
715 void **pp = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +0000716 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800717 page_flush_tb_1 (level - 1, pp + i);
718 }
719 }
720}
721
bellardfd6ce8f2003-05-14 19:00:11 +0000722static void page_flush_tb(void)
723{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800724 int i;
725 for (i = 0; i < V_L1_SIZE; i++) {
726 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
bellardfd6ce8f2003-05-14 19:00:11 +0000727 }
728}
729
730/* flush all the translation blocks */
bellardd4e81642003-05-25 16:46:15 +0000731/* XXX: tb_flush is currently not thread safe */
bellard6a00d602005-11-21 23:25:50 +0000732void tb_flush(CPUState *env1)
bellardfd6ce8f2003-05-14 19:00:11 +0000733{
bellard6a00d602005-11-21 23:25:50 +0000734 CPUState *env;
bellard01243112004-01-04 15:48:17 +0000735#if defined(DEBUG_FLUSH)
blueswir1ab3d1722007-11-04 07:31:40 +0000736 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
737 (unsigned long)(code_gen_ptr - code_gen_buffer),
738 nb_tbs, nb_tbs > 0 ?
739 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
bellardfd6ce8f2003-05-14 19:00:11 +0000740#endif
bellard26a5f132008-05-28 12:30:31 +0000741 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
pbrooka208e542008-03-31 17:07:36 +0000742 cpu_abort(env1, "Internal error: code buffer overflow\n");
743
bellardfd6ce8f2003-05-14 19:00:11 +0000744 nb_tbs = 0;
ths3b46e622007-09-17 08:09:54 +0000745
bellard6a00d602005-11-21 23:25:50 +0000746 for(env = first_cpu; env != NULL; env = env->next_cpu) {
747 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
748 }
bellard9fa3e852004-01-04 18:06:42 +0000749
bellard8a8a6082004-10-03 13:36:49 +0000750 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
bellardfd6ce8f2003-05-14 19:00:11 +0000751 page_flush_tb();
bellard9fa3e852004-01-04 18:06:42 +0000752
bellardfd6ce8f2003-05-14 19:00:11 +0000753 code_gen_ptr = code_gen_buffer;
bellardd4e81642003-05-25 16:46:15 +0000754 /* XXX: flush processor icache at this point if cache flush is
755 expensive */
bellarde3db7222005-01-26 22:00:47 +0000756 tb_flush_count++;
bellardfd6ce8f2003-05-14 19:00:11 +0000757}
758
759#ifdef DEBUG_TB_CHECK
760
j_mayerbc98a7e2007-04-04 07:55:12 +0000761static void tb_invalidate_check(target_ulong address)
bellardfd6ce8f2003-05-14 19:00:11 +0000762{
763 TranslationBlock *tb;
764 int i;
765 address &= TARGET_PAGE_MASK;
pbrook99773bd2006-04-16 15:14:59 +0000766 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
767 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
bellardfd6ce8f2003-05-14 19:00:11 +0000768 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
769 address >= tb->pc + tb->size)) {
Blue Swirl0bf9e312009-07-20 17:19:25 +0000770 printf("ERROR invalidate: address=" TARGET_FMT_lx
771 " PC=%08lx size=%04x\n",
pbrook99773bd2006-04-16 15:14:59 +0000772 address, (long)tb->pc, tb->size);
bellardfd6ce8f2003-05-14 19:00:11 +0000773 }
774 }
775 }
776}
777
778/* verify that all the pages have correct rights for code */
779static void tb_page_check(void)
780{
781 TranslationBlock *tb;
782 int i, flags1, flags2;
ths3b46e622007-09-17 08:09:54 +0000783
pbrook99773bd2006-04-16 15:14:59 +0000784 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
785 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
bellardfd6ce8f2003-05-14 19:00:11 +0000786 flags1 = page_get_flags(tb->pc);
787 flags2 = page_get_flags(tb->pc + tb->size - 1);
788 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
789 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
pbrook99773bd2006-04-16 15:14:59 +0000790 (long)tb->pc, tb->size, flags1, flags2);
bellardfd6ce8f2003-05-14 19:00:11 +0000791 }
792 }
793 }
794}
795
796#endif
797
798/* invalidate one TB */
799static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
800 int next_offset)
801{
802 TranslationBlock *tb1;
803 for(;;) {
804 tb1 = *ptb;
805 if (tb1 == tb) {
806 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
807 break;
808 }
809 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
810 }
811}
812
bellard9fa3e852004-01-04 18:06:42 +0000813static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
814{
815 TranslationBlock *tb1;
816 unsigned int n1;
817
818 for(;;) {
819 tb1 = *ptb;
820 n1 = (long)tb1 & 3;
821 tb1 = (TranslationBlock *)((long)tb1 & ~3);
822 if (tb1 == tb) {
823 *ptb = tb1->page_next[n1];
824 break;
825 }
826 ptb = &tb1->page_next[n1];
827 }
828}
829
bellardd4e81642003-05-25 16:46:15 +0000830static inline void tb_jmp_remove(TranslationBlock *tb, int n)
831{
832 TranslationBlock *tb1, **ptb;
833 unsigned int n1;
834
835 ptb = &tb->jmp_next[n];
836 tb1 = *ptb;
837 if (tb1) {
838 /* find tb(n) in circular list */
839 for(;;) {
840 tb1 = *ptb;
841 n1 = (long)tb1 & 3;
842 tb1 = (TranslationBlock *)((long)tb1 & ~3);
843 if (n1 == n && tb1 == tb)
844 break;
845 if (n1 == 2) {
846 ptb = &tb1->jmp_first;
847 } else {
848 ptb = &tb1->jmp_next[n1];
849 }
850 }
851 /* now we can suppress tb(n) from the list */
852 *ptb = tb->jmp_next[n];
853
854 tb->jmp_next[n] = NULL;
855 }
856}
857
858/* reset the jump entry 'n' of a TB so that it is not chained to
859 another TB */
860static inline void tb_reset_jump(TranslationBlock *tb, int n)
861{
862 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
863}
864
Paul Brook41c1b1c2010-03-12 16:54:58 +0000865void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
bellardfd6ce8f2003-05-14 19:00:11 +0000866{
bellard6a00d602005-11-21 23:25:50 +0000867 CPUState *env;
bellardfd6ce8f2003-05-14 19:00:11 +0000868 PageDesc *p;
bellard8a40a182005-11-20 10:35:40 +0000869 unsigned int h, n1;
Paul Brook41c1b1c2010-03-12 16:54:58 +0000870 tb_page_addr_t phys_pc;
bellard8a40a182005-11-20 10:35:40 +0000871 TranslationBlock *tb1, *tb2;
ths3b46e622007-09-17 08:09:54 +0000872
bellard9fa3e852004-01-04 18:06:42 +0000873 /* remove the TB from the hash list */
874 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
875 h = tb_phys_hash_func(phys_pc);
ths5fafdf22007-09-16 21:08:06 +0000876 tb_remove(&tb_phys_hash[h], tb,
bellard9fa3e852004-01-04 18:06:42 +0000877 offsetof(TranslationBlock, phys_hash_next));
bellardfd6ce8f2003-05-14 19:00:11 +0000878
bellard9fa3e852004-01-04 18:06:42 +0000879 /* remove the TB from the page list */
880 if (tb->page_addr[0] != page_addr) {
881 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
882 tb_page_remove(&p->first_tb, tb);
883 invalidate_page_bitmap(p);
884 }
885 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
886 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
887 tb_page_remove(&p->first_tb, tb);
888 invalidate_page_bitmap(p);
889 }
890
bellard8a40a182005-11-20 10:35:40 +0000891 tb_invalidated_flag = 1;
892
893 /* remove the TB from the hash list */
894 h = tb_jmp_cache_hash_func(tb->pc);
bellard6a00d602005-11-21 23:25:50 +0000895 for(env = first_cpu; env != NULL; env = env->next_cpu) {
896 if (env->tb_jmp_cache[h] == tb)
897 env->tb_jmp_cache[h] = NULL;
898 }
bellard8a40a182005-11-20 10:35:40 +0000899
900 /* suppress this TB from the two jump lists */
901 tb_jmp_remove(tb, 0);
902 tb_jmp_remove(tb, 1);
903
904 /* suppress any remaining jumps to this TB */
905 tb1 = tb->jmp_first;
906 for(;;) {
907 n1 = (long)tb1 & 3;
908 if (n1 == 2)
909 break;
910 tb1 = (TranslationBlock *)((long)tb1 & ~3);
911 tb2 = tb1->jmp_next[n1];
912 tb_reset_jump(tb1, n1);
913 tb1->jmp_next[n1] = NULL;
914 tb1 = tb2;
915 }
916 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
917
bellarde3db7222005-01-26 22:00:47 +0000918 tb_phys_invalidate_count++;
bellard9fa3e852004-01-04 18:06:42 +0000919}
920
921static inline void set_bits(uint8_t *tab, int start, int len)
922{
923 int end, mask, end1;
924
925 end = start + len;
926 tab += start >> 3;
927 mask = 0xff << (start & 7);
928 if ((start & ~7) == (end & ~7)) {
929 if (start < end) {
930 mask &= ~(0xff << (end & 7));
931 *tab |= mask;
932 }
933 } else {
934 *tab++ |= mask;
935 start = (start + 8) & ~7;
936 end1 = end & ~7;
937 while (start < end1) {
938 *tab++ = 0xff;
939 start += 8;
940 }
941 if (start < end) {
942 mask = ~(0xff << (end & 7));
943 *tab |= mask;
944 }
945 }
946}
947
948static void build_page_bitmap(PageDesc *p)
949{
950 int n, tb_start, tb_end;
951 TranslationBlock *tb;
ths3b46e622007-09-17 08:09:54 +0000952
pbrookb2a70812008-06-09 13:57:23 +0000953 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
bellard9fa3e852004-01-04 18:06:42 +0000954
955 tb = p->first_tb;
956 while (tb != NULL) {
957 n = (long)tb & 3;
958 tb = (TranslationBlock *)((long)tb & ~3);
959 /* NOTE: this is subtle as a TB may span two physical pages */
960 if (n == 0) {
961 /* NOTE: tb_end may be after the end of the page, but
962 it is not a problem */
963 tb_start = tb->pc & ~TARGET_PAGE_MASK;
964 tb_end = tb_start + tb->size;
965 if (tb_end > TARGET_PAGE_SIZE)
966 tb_end = TARGET_PAGE_SIZE;
967 } else {
968 tb_start = 0;
969 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
970 }
971 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
972 tb = tb->page_next[n];
973 }
974}
975
pbrook2e70f6e2008-06-29 01:03:05 +0000976TranslationBlock *tb_gen_code(CPUState *env,
977 target_ulong pc, target_ulong cs_base,
978 int flags, int cflags)
bellardd720b932004-04-25 17:57:43 +0000979{
980 TranslationBlock *tb;
981 uint8_t *tc_ptr;
Paul Brook41c1b1c2010-03-12 16:54:58 +0000982 tb_page_addr_t phys_pc, phys_page2;
983 target_ulong virt_page2;
bellardd720b932004-04-25 17:57:43 +0000984 int code_gen_size;
985
Paul Brook41c1b1c2010-03-12 16:54:58 +0000986 phys_pc = get_page_addr_code(env, pc);
bellardc27004e2005-01-03 23:35:10 +0000987 tb = tb_alloc(pc);
bellardd720b932004-04-25 17:57:43 +0000988 if (!tb) {
989 /* flush must be done */
990 tb_flush(env);
991 /* cannot fail at this point */
bellardc27004e2005-01-03 23:35:10 +0000992 tb = tb_alloc(pc);
pbrook2e70f6e2008-06-29 01:03:05 +0000993 /* Don't forget to invalidate previous TB info. */
994 tb_invalidated_flag = 1;
bellardd720b932004-04-25 17:57:43 +0000995 }
996 tc_ptr = code_gen_ptr;
997 tb->tc_ptr = tc_ptr;
998 tb->cs_base = cs_base;
999 tb->flags = flags;
1000 tb->cflags = cflags;
blueswir1d07bde82007-12-11 19:35:45 +00001001 cpu_gen_code(env, tb, &code_gen_size);
bellardd720b932004-04-25 17:57:43 +00001002 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
ths3b46e622007-09-17 08:09:54 +00001003
bellardd720b932004-04-25 17:57:43 +00001004 /* check next page if needed */
bellardc27004e2005-01-03 23:35:10 +00001005 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
bellardd720b932004-04-25 17:57:43 +00001006 phys_page2 = -1;
bellardc27004e2005-01-03 23:35:10 +00001007 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
Paul Brook41c1b1c2010-03-12 16:54:58 +00001008 phys_page2 = get_page_addr_code(env, virt_page2);
bellardd720b932004-04-25 17:57:43 +00001009 }
Paul Brook41c1b1c2010-03-12 16:54:58 +00001010 tb_link_page(tb, phys_pc, phys_page2);
pbrook2e70f6e2008-06-29 01:03:05 +00001011 return tb;
bellardd720b932004-04-25 17:57:43 +00001012}
ths3b46e622007-09-17 08:09:54 +00001013
bellard9fa3e852004-01-04 18:06:42 +00001014/* invalidate all TBs which intersect with the target physical page
1015 starting in range [start;end[. NOTE: start and end must refer to
bellardd720b932004-04-25 17:57:43 +00001016 the same physical page. 'is_cpu_write_access' should be true if called
1017 from a real cpu write access: the virtual CPU will exit the current
1018 TB if code is modified inside this TB. */
Paul Brook41c1b1c2010-03-12 16:54:58 +00001019void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
bellardd720b932004-04-25 17:57:43 +00001020 int is_cpu_write_access)
bellard9fa3e852004-01-04 18:06:42 +00001021{
aliguori6b917542008-11-18 19:46:41 +00001022 TranslationBlock *tb, *tb_next, *saved_tb;
bellardd720b932004-04-25 17:57:43 +00001023 CPUState *env = cpu_single_env;
Paul Brook41c1b1c2010-03-12 16:54:58 +00001024 tb_page_addr_t tb_start, tb_end;
aliguori6b917542008-11-18 19:46:41 +00001025 PageDesc *p;
1026 int n;
1027#ifdef TARGET_HAS_PRECISE_SMC
1028 int current_tb_not_found = is_cpu_write_access;
1029 TranslationBlock *current_tb = NULL;
1030 int current_tb_modified = 0;
1031 target_ulong current_pc = 0;
1032 target_ulong current_cs_base = 0;
1033 int current_flags = 0;
1034#endif /* TARGET_HAS_PRECISE_SMC */
bellard9fa3e852004-01-04 18:06:42 +00001035
1036 p = page_find(start >> TARGET_PAGE_BITS);
ths5fafdf22007-09-16 21:08:06 +00001037 if (!p)
bellard9fa3e852004-01-04 18:06:42 +00001038 return;
ths5fafdf22007-09-16 21:08:06 +00001039 if (!p->code_bitmap &&
bellardd720b932004-04-25 17:57:43 +00001040 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1041 is_cpu_write_access) {
bellard9fa3e852004-01-04 18:06:42 +00001042 /* build code bitmap */
1043 build_page_bitmap(p);
1044 }
1045
1046 /* we remove all the TBs in the range [start, end[ */
1047 /* XXX: see if in some cases it could be faster to invalidate all the code */
1048 tb = p->first_tb;
1049 while (tb != NULL) {
1050 n = (long)tb & 3;
1051 tb = (TranslationBlock *)((long)tb & ~3);
1052 tb_next = tb->page_next[n];
1053 /* NOTE: this is subtle as a TB may span two physical pages */
1054 if (n == 0) {
1055 /* NOTE: tb_end may be after the end of the page, but
1056 it is not a problem */
1057 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1058 tb_end = tb_start + tb->size;
1059 } else {
1060 tb_start = tb->page_addr[1];
1061 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1062 }
1063 if (!(tb_end <= start || tb_start >= end)) {
bellardd720b932004-04-25 17:57:43 +00001064#ifdef TARGET_HAS_PRECISE_SMC
1065 if (current_tb_not_found) {
1066 current_tb_not_found = 0;
1067 current_tb = NULL;
pbrook2e70f6e2008-06-29 01:03:05 +00001068 if (env->mem_io_pc) {
bellardd720b932004-04-25 17:57:43 +00001069 /* now we have a real cpu fault */
pbrook2e70f6e2008-06-29 01:03:05 +00001070 current_tb = tb_find_pc(env->mem_io_pc);
bellardd720b932004-04-25 17:57:43 +00001071 }
1072 }
1073 if (current_tb == tb &&
pbrook2e70f6e2008-06-29 01:03:05 +00001074 (current_tb->cflags & CF_COUNT_MASK) != 1) {
bellardd720b932004-04-25 17:57:43 +00001075 /* If we are modifying the current TB, we must stop
1076 its execution. We could be more precise by checking
1077 that the modification is after the current PC, but it
1078 would require a specialized function to partially
1079 restore the CPU state */
ths3b46e622007-09-17 08:09:54 +00001080
bellardd720b932004-04-25 17:57:43 +00001081 current_tb_modified = 1;
Stefan Weil618ba8e2011-04-18 06:39:53 +00001082 cpu_restore_state(current_tb, env, env->mem_io_pc);
aliguori6b917542008-11-18 19:46:41 +00001083 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1084 &current_flags);
bellardd720b932004-04-25 17:57:43 +00001085 }
1086#endif /* TARGET_HAS_PRECISE_SMC */
bellard6f5a9f72005-11-26 20:12:28 +00001087 /* we need to do that to handle the case where a signal
1088 occurs while doing tb_phys_invalidate() */
1089 saved_tb = NULL;
1090 if (env) {
1091 saved_tb = env->current_tb;
1092 env->current_tb = NULL;
1093 }
bellard9fa3e852004-01-04 18:06:42 +00001094 tb_phys_invalidate(tb, -1);
bellard6f5a9f72005-11-26 20:12:28 +00001095 if (env) {
1096 env->current_tb = saved_tb;
1097 if (env->interrupt_request && env->current_tb)
1098 cpu_interrupt(env, env->interrupt_request);
1099 }
bellard9fa3e852004-01-04 18:06:42 +00001100 }
1101 tb = tb_next;
1102 }
1103#if !defined(CONFIG_USER_ONLY)
1104 /* if no code remaining, no need to continue to use slow writes */
1105 if (!p->first_tb) {
1106 invalidate_page_bitmap(p);
bellardd720b932004-04-25 17:57:43 +00001107 if (is_cpu_write_access) {
pbrook2e70f6e2008-06-29 01:03:05 +00001108 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
bellardd720b932004-04-25 17:57:43 +00001109 }
1110 }
1111#endif
1112#ifdef TARGET_HAS_PRECISE_SMC
1113 if (current_tb_modified) {
1114 /* we generate a block containing just the instruction
1115 modifying the memory. It will ensure that it cannot modify
1116 itself */
bellardea1c1802004-06-14 18:56:36 +00001117 env->current_tb = NULL;
pbrook2e70f6e2008-06-29 01:03:05 +00001118 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
bellardd720b932004-04-25 17:57:43 +00001119 cpu_resume_from_signal(env, NULL);
bellard9fa3e852004-01-04 18:06:42 +00001120 }
1121#endif
1122}
1123
1124/* len must be <= 8 and start must be a multiple of len */
Paul Brook41c1b1c2010-03-12 16:54:58 +00001125static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
bellard9fa3e852004-01-04 18:06:42 +00001126{
1127 PageDesc *p;
1128 int offset, b;
bellard59817cc2004-02-16 22:01:13 +00001129#if 0
bellarda4193c82004-06-03 14:01:43 +00001130 if (1) {
aliguori93fcfe32009-01-15 22:34:14 +00001131 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1132 cpu_single_env->mem_io_vaddr, len,
1133 cpu_single_env->eip,
1134 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
bellard59817cc2004-02-16 22:01:13 +00001135 }
1136#endif
bellard9fa3e852004-01-04 18:06:42 +00001137 p = page_find(start >> TARGET_PAGE_BITS);
ths5fafdf22007-09-16 21:08:06 +00001138 if (!p)
bellard9fa3e852004-01-04 18:06:42 +00001139 return;
1140 if (p->code_bitmap) {
1141 offset = start & ~TARGET_PAGE_MASK;
1142 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1143 if (b & ((1 << len) - 1))
1144 goto do_invalidate;
1145 } else {
1146 do_invalidate:
bellardd720b932004-04-25 17:57:43 +00001147 tb_invalidate_phys_page_range(start, start + len, 1);
bellard9fa3e852004-01-04 18:06:42 +00001148 }
1149}
1150
bellard9fa3e852004-01-04 18:06:42 +00001151#if !defined(CONFIG_SOFTMMU)
Paul Brook41c1b1c2010-03-12 16:54:58 +00001152static void tb_invalidate_phys_page(tb_page_addr_t addr,
bellardd720b932004-04-25 17:57:43 +00001153 unsigned long pc, void *puc)
bellard9fa3e852004-01-04 18:06:42 +00001154{
aliguori6b917542008-11-18 19:46:41 +00001155 TranslationBlock *tb;
bellard9fa3e852004-01-04 18:06:42 +00001156 PageDesc *p;
aliguori6b917542008-11-18 19:46:41 +00001157 int n;
bellardd720b932004-04-25 17:57:43 +00001158#ifdef TARGET_HAS_PRECISE_SMC
aliguori6b917542008-11-18 19:46:41 +00001159 TranslationBlock *current_tb = NULL;
bellardd720b932004-04-25 17:57:43 +00001160 CPUState *env = cpu_single_env;
aliguori6b917542008-11-18 19:46:41 +00001161 int current_tb_modified = 0;
1162 target_ulong current_pc = 0;
1163 target_ulong current_cs_base = 0;
1164 int current_flags = 0;
bellardd720b932004-04-25 17:57:43 +00001165#endif
bellard9fa3e852004-01-04 18:06:42 +00001166
1167 addr &= TARGET_PAGE_MASK;
1168 p = page_find(addr >> TARGET_PAGE_BITS);
ths5fafdf22007-09-16 21:08:06 +00001169 if (!p)
bellardfd6ce8f2003-05-14 19:00:11 +00001170 return;
1171 tb = p->first_tb;
bellardd720b932004-04-25 17:57:43 +00001172#ifdef TARGET_HAS_PRECISE_SMC
1173 if (tb && pc != 0) {
1174 current_tb = tb_find_pc(pc);
1175 }
1176#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001177 while (tb != NULL) {
bellard9fa3e852004-01-04 18:06:42 +00001178 n = (long)tb & 3;
1179 tb = (TranslationBlock *)((long)tb & ~3);
bellardd720b932004-04-25 17:57:43 +00001180#ifdef TARGET_HAS_PRECISE_SMC
1181 if (current_tb == tb &&
pbrook2e70f6e2008-06-29 01:03:05 +00001182 (current_tb->cflags & CF_COUNT_MASK) != 1) {
bellardd720b932004-04-25 17:57:43 +00001183 /* If we are modifying the current TB, we must stop
1184 its execution. We could be more precise by checking
1185 that the modification is after the current PC, but it
1186 would require a specialized function to partially
1187 restore the CPU state */
ths3b46e622007-09-17 08:09:54 +00001188
bellardd720b932004-04-25 17:57:43 +00001189 current_tb_modified = 1;
Stefan Weil618ba8e2011-04-18 06:39:53 +00001190 cpu_restore_state(current_tb, env, pc);
aliguori6b917542008-11-18 19:46:41 +00001191 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1192 &current_flags);
bellardd720b932004-04-25 17:57:43 +00001193 }
1194#endif /* TARGET_HAS_PRECISE_SMC */
bellard9fa3e852004-01-04 18:06:42 +00001195 tb_phys_invalidate(tb, addr);
1196 tb = tb->page_next[n];
bellardfd6ce8f2003-05-14 19:00:11 +00001197 }
1198 p->first_tb = NULL;
bellardd720b932004-04-25 17:57:43 +00001199#ifdef TARGET_HAS_PRECISE_SMC
1200 if (current_tb_modified) {
1201 /* we generate a block containing just the instruction
1202 modifying the memory. It will ensure that it cannot modify
1203 itself */
bellardea1c1802004-06-14 18:56:36 +00001204 env->current_tb = NULL;
pbrook2e70f6e2008-06-29 01:03:05 +00001205 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
bellardd720b932004-04-25 17:57:43 +00001206 cpu_resume_from_signal(env, puc);
1207 }
1208#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001209}
bellard9fa3e852004-01-04 18:06:42 +00001210#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001211
1212/* add the tb in the target page and protect it if necessary */
ths5fafdf22007-09-16 21:08:06 +00001213static inline void tb_alloc_page(TranslationBlock *tb,
Paul Brook41c1b1c2010-03-12 16:54:58 +00001214 unsigned int n, tb_page_addr_t page_addr)
bellardfd6ce8f2003-05-14 19:00:11 +00001215{
1216 PageDesc *p;
Juan Quintela4429ab42011-06-02 01:53:44 +00001217#ifndef CONFIG_USER_ONLY
1218 bool page_already_protected;
1219#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001220
bellard9fa3e852004-01-04 18:06:42 +00001221 tb->page_addr[n] = page_addr;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001222 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
bellard9fa3e852004-01-04 18:06:42 +00001223 tb->page_next[n] = p->first_tb;
Juan Quintela4429ab42011-06-02 01:53:44 +00001224#ifndef CONFIG_USER_ONLY
1225 page_already_protected = p->first_tb != NULL;
1226#endif
bellard9fa3e852004-01-04 18:06:42 +00001227 p->first_tb = (TranslationBlock *)((long)tb | n);
1228 invalidate_page_bitmap(p);
1229
bellard107db442004-06-22 18:48:46 +00001230#if defined(TARGET_HAS_SMC) || 1
bellardd720b932004-04-25 17:57:43 +00001231
bellard9fa3e852004-01-04 18:06:42 +00001232#if defined(CONFIG_USER_ONLY)
bellardfd6ce8f2003-05-14 19:00:11 +00001233 if (p->flags & PAGE_WRITE) {
pbrook53a59602006-03-25 19:31:22 +00001234 target_ulong addr;
1235 PageDesc *p2;
bellard9fa3e852004-01-04 18:06:42 +00001236 int prot;
1237
bellardfd6ce8f2003-05-14 19:00:11 +00001238 /* force the host page as non writable (writes will have a
1239 page fault + mprotect overhead) */
pbrook53a59602006-03-25 19:31:22 +00001240 page_addr &= qemu_host_page_mask;
bellardfd6ce8f2003-05-14 19:00:11 +00001241 prot = 0;
pbrook53a59602006-03-25 19:31:22 +00001242 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1243 addr += TARGET_PAGE_SIZE) {
1244
1245 p2 = page_find (addr >> TARGET_PAGE_BITS);
1246 if (!p2)
1247 continue;
1248 prot |= p2->flags;
1249 p2->flags &= ~PAGE_WRITE;
pbrook53a59602006-03-25 19:31:22 +00001250 }
ths5fafdf22007-09-16 21:08:06 +00001251 mprotect(g2h(page_addr), qemu_host_page_size,
bellardfd6ce8f2003-05-14 19:00:11 +00001252 (prot & PAGE_BITS) & ~PAGE_WRITE);
1253#ifdef DEBUG_TB_INVALIDATE
blueswir1ab3d1722007-11-04 07:31:40 +00001254 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
pbrook53a59602006-03-25 19:31:22 +00001255 page_addr);
bellardfd6ce8f2003-05-14 19:00:11 +00001256#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001257 }
bellard9fa3e852004-01-04 18:06:42 +00001258#else
1259 /* if some code is already present, then the pages are already
1260 protected. So we handle the case where only the first TB is
1261 allocated in a physical page */
Juan Quintela4429ab42011-06-02 01:53:44 +00001262 if (!page_already_protected) {
bellard6a00d602005-11-21 23:25:50 +00001263 tlb_protect_code(page_addr);
bellard9fa3e852004-01-04 18:06:42 +00001264 }
1265#endif
bellardd720b932004-04-25 17:57:43 +00001266
1267#endif /* TARGET_HAS_SMC */
bellardfd6ce8f2003-05-14 19:00:11 +00001268}
1269
bellard9fa3e852004-01-04 18:06:42 +00001270/* add a new TB and link it to the physical page tables. phys_page2 is
1271 (-1) to indicate that only one page contains the TB. */
Paul Brook41c1b1c2010-03-12 16:54:58 +00001272void tb_link_page(TranslationBlock *tb,
1273 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
bellardd4e81642003-05-25 16:46:15 +00001274{
bellard9fa3e852004-01-04 18:06:42 +00001275 unsigned int h;
1276 TranslationBlock **ptb;
1277
pbrookc8a706f2008-06-02 16:16:42 +00001278 /* Grab the mmap lock to stop another thread invalidating this TB
1279 before we are done. */
1280 mmap_lock();
bellard9fa3e852004-01-04 18:06:42 +00001281 /* add in the physical hash table */
1282 h = tb_phys_hash_func(phys_pc);
1283 ptb = &tb_phys_hash[h];
1284 tb->phys_hash_next = *ptb;
1285 *ptb = tb;
bellardfd6ce8f2003-05-14 19:00:11 +00001286
1287 /* add in the page list */
bellard9fa3e852004-01-04 18:06:42 +00001288 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1289 if (phys_page2 != -1)
1290 tb_alloc_page(tb, 1, phys_page2);
1291 else
1292 tb->page_addr[1] = -1;
bellard9fa3e852004-01-04 18:06:42 +00001293
bellardd4e81642003-05-25 16:46:15 +00001294 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1295 tb->jmp_next[0] = NULL;
1296 tb->jmp_next[1] = NULL;
1297
1298 /* init original jump addresses */
1299 if (tb->tb_next_offset[0] != 0xffff)
1300 tb_reset_jump(tb, 0);
1301 if (tb->tb_next_offset[1] != 0xffff)
1302 tb_reset_jump(tb, 1);
bellard8a40a182005-11-20 10:35:40 +00001303
1304#ifdef DEBUG_TB_CHECK
1305 tb_page_check();
1306#endif
pbrookc8a706f2008-06-02 16:16:42 +00001307 mmap_unlock();
bellardfd6ce8f2003-05-14 19:00:11 +00001308}
1309
bellarda513fe12003-05-27 23:29:48 +00001310/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1311 tb[1].tc_ptr. Return NULL if not found */
1312TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1313{
1314 int m_min, m_max, m;
1315 unsigned long v;
1316 TranslationBlock *tb;
1317
1318 if (nb_tbs <= 0)
1319 return NULL;
1320 if (tc_ptr < (unsigned long)code_gen_buffer ||
1321 tc_ptr >= (unsigned long)code_gen_ptr)
1322 return NULL;
1323 /* binary search (cf Knuth) */
1324 m_min = 0;
1325 m_max = nb_tbs - 1;
1326 while (m_min <= m_max) {
1327 m = (m_min + m_max) >> 1;
1328 tb = &tbs[m];
1329 v = (unsigned long)tb->tc_ptr;
1330 if (v == tc_ptr)
1331 return tb;
1332 else if (tc_ptr < v) {
1333 m_max = m - 1;
1334 } else {
1335 m_min = m + 1;
1336 }
ths5fafdf22007-09-16 21:08:06 +00001337 }
bellarda513fe12003-05-27 23:29:48 +00001338 return &tbs[m_max];
1339}
bellard75012672003-06-21 13:11:07 +00001340
bellardea041c02003-06-25 16:16:50 +00001341static void tb_reset_jump_recursive(TranslationBlock *tb);
1342
1343static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1344{
1345 TranslationBlock *tb1, *tb_next, **ptb;
1346 unsigned int n1;
1347
1348 tb1 = tb->jmp_next[n];
1349 if (tb1 != NULL) {
1350 /* find head of list */
1351 for(;;) {
1352 n1 = (long)tb1 & 3;
1353 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1354 if (n1 == 2)
1355 break;
1356 tb1 = tb1->jmp_next[n1];
1357 }
1358 /* we are now sure now that tb jumps to tb1 */
1359 tb_next = tb1;
1360
1361 /* remove tb from the jmp_first list */
1362 ptb = &tb_next->jmp_first;
1363 for(;;) {
1364 tb1 = *ptb;
1365 n1 = (long)tb1 & 3;
1366 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1367 if (n1 == n && tb1 == tb)
1368 break;
1369 ptb = &tb1->jmp_next[n1];
1370 }
1371 *ptb = tb->jmp_next[n];
1372 tb->jmp_next[n] = NULL;
ths3b46e622007-09-17 08:09:54 +00001373
bellardea041c02003-06-25 16:16:50 +00001374 /* suppress the jump to next tb in generated code */
1375 tb_reset_jump(tb, n);
1376
bellard01243112004-01-04 15:48:17 +00001377 /* suppress jumps in the tb on which we could have jumped */
bellardea041c02003-06-25 16:16:50 +00001378 tb_reset_jump_recursive(tb_next);
1379 }
1380}
1381
1382static void tb_reset_jump_recursive(TranslationBlock *tb)
1383{
1384 tb_reset_jump_recursive2(tb, 0);
1385 tb_reset_jump_recursive2(tb, 1);
1386}
1387
bellard1fddef42005-04-17 19:16:13 +00001388#if defined(TARGET_HAS_ICE)
Paul Brook94df27f2010-02-28 23:47:45 +00001389#if defined(CONFIG_USER_ONLY)
1390static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1391{
1392 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1393}
1394#else
bellardd720b932004-04-25 17:57:43 +00001395static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1396{
Anthony Liguoric227f092009-10-01 16:12:16 -05001397 target_phys_addr_t addr;
j_mayer9b3c35e2007-04-07 11:21:28 +00001398 target_ulong pd;
Anthony Liguoric227f092009-10-01 16:12:16 -05001399 ram_addr_t ram_addr;
pbrookc2f07f82006-04-08 17:14:56 +00001400 PhysPageDesc *p;
bellardd720b932004-04-25 17:57:43 +00001401
pbrookc2f07f82006-04-08 17:14:56 +00001402 addr = cpu_get_phys_page_debug(env, pc);
1403 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1404 if (!p) {
1405 pd = IO_MEM_UNASSIGNED;
1406 } else {
1407 pd = p->phys_offset;
1408 }
1409 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
pbrook706cd4b2006-04-08 17:36:21 +00001410 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
bellardd720b932004-04-25 17:57:43 +00001411}
bellardc27004e2005-01-03 23:35:10 +00001412#endif
Paul Brook94df27f2010-02-28 23:47:45 +00001413#endif /* TARGET_HAS_ICE */
bellardd720b932004-04-25 17:57:43 +00001414
Paul Brookc527ee82010-03-01 03:31:14 +00001415#if defined(CONFIG_USER_ONLY)
1416void cpu_watchpoint_remove_all(CPUState *env, int mask)
1417
1418{
1419}
1420
1421int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1422 int flags, CPUWatchpoint **watchpoint)
1423{
1424 return -ENOSYS;
1425}
1426#else
pbrook6658ffb2007-03-16 23:58:11 +00001427/* Add a watchpoint. */
aliguoria1d1bb32008-11-18 20:07:32 +00001428int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1429 int flags, CPUWatchpoint **watchpoint)
pbrook6658ffb2007-03-16 23:58:11 +00001430{
aliguorib4051332008-11-18 20:14:20 +00001431 target_ulong len_mask = ~(len - 1);
aliguoric0ce9982008-11-25 22:13:57 +00001432 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +00001433
aliguorib4051332008-11-18 20:14:20 +00001434 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1435 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1436 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1437 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1438 return -EINVAL;
1439 }
aliguoria1d1bb32008-11-18 20:07:32 +00001440 wp = qemu_malloc(sizeof(*wp));
pbrook6658ffb2007-03-16 23:58:11 +00001441
aliguoria1d1bb32008-11-18 20:07:32 +00001442 wp->vaddr = addr;
aliguorib4051332008-11-18 20:14:20 +00001443 wp->len_mask = len_mask;
aliguoria1d1bb32008-11-18 20:07:32 +00001444 wp->flags = flags;
1445
aliguori2dc9f412008-11-18 20:56:59 +00001446 /* keep all GDB-injected watchpoints in front */
aliguoric0ce9982008-11-25 22:13:57 +00001447 if (flags & BP_GDB)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001448 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
aliguoric0ce9982008-11-25 22:13:57 +00001449 else
Blue Swirl72cf2d42009-09-12 07:36:22 +00001450 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
aliguoria1d1bb32008-11-18 20:07:32 +00001451
pbrook6658ffb2007-03-16 23:58:11 +00001452 tlb_flush_page(env, addr);
aliguoria1d1bb32008-11-18 20:07:32 +00001453
1454 if (watchpoint)
1455 *watchpoint = wp;
1456 return 0;
pbrook6658ffb2007-03-16 23:58:11 +00001457}
1458
aliguoria1d1bb32008-11-18 20:07:32 +00001459/* Remove a specific watchpoint. */
1460int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1461 int flags)
pbrook6658ffb2007-03-16 23:58:11 +00001462{
aliguorib4051332008-11-18 20:14:20 +00001463 target_ulong len_mask = ~(len - 1);
aliguoria1d1bb32008-11-18 20:07:32 +00001464 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +00001465
Blue Swirl72cf2d42009-09-12 07:36:22 +00001466 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguorib4051332008-11-18 20:14:20 +00001467 if (addr == wp->vaddr && len_mask == wp->len_mask
aliguori6e140f22008-11-18 20:37:55 +00001468 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
aliguoria1d1bb32008-11-18 20:07:32 +00001469 cpu_watchpoint_remove_by_ref(env, wp);
pbrook6658ffb2007-03-16 23:58:11 +00001470 return 0;
1471 }
1472 }
aliguoria1d1bb32008-11-18 20:07:32 +00001473 return -ENOENT;
pbrook6658ffb2007-03-16 23:58:11 +00001474}
1475
aliguoria1d1bb32008-11-18 20:07:32 +00001476/* Remove a specific watchpoint by reference. */
1477void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1478{
Blue Swirl72cf2d42009-09-12 07:36:22 +00001479 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
edgar_igl7d03f822008-05-17 18:58:29 +00001480
aliguoria1d1bb32008-11-18 20:07:32 +00001481 tlb_flush_page(env, watchpoint->vaddr);
1482
1483 qemu_free(watchpoint);
edgar_igl7d03f822008-05-17 18:58:29 +00001484}
1485
aliguoria1d1bb32008-11-18 20:07:32 +00001486/* Remove all matching watchpoints. */
1487void cpu_watchpoint_remove_all(CPUState *env, int mask)
1488{
aliguoric0ce9982008-11-25 22:13:57 +00001489 CPUWatchpoint *wp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +00001490
Blue Swirl72cf2d42009-09-12 07:36:22 +00001491 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
aliguoria1d1bb32008-11-18 20:07:32 +00001492 if (wp->flags & mask)
1493 cpu_watchpoint_remove_by_ref(env, wp);
aliguoric0ce9982008-11-25 22:13:57 +00001494 }
aliguoria1d1bb32008-11-18 20:07:32 +00001495}
Paul Brookc527ee82010-03-01 03:31:14 +00001496#endif
aliguoria1d1bb32008-11-18 20:07:32 +00001497
1498/* Add a breakpoint. */
1499int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1500 CPUBreakpoint **breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +00001501{
bellard1fddef42005-04-17 19:16:13 +00001502#if defined(TARGET_HAS_ICE)
aliguoric0ce9982008-11-25 22:13:57 +00001503 CPUBreakpoint *bp;
ths3b46e622007-09-17 08:09:54 +00001504
aliguoria1d1bb32008-11-18 20:07:32 +00001505 bp = qemu_malloc(sizeof(*bp));
aliguoria1d1bb32008-11-18 20:07:32 +00001506
1507 bp->pc = pc;
1508 bp->flags = flags;
1509
aliguori2dc9f412008-11-18 20:56:59 +00001510 /* keep all GDB-injected breakpoints in front */
aliguoric0ce9982008-11-25 22:13:57 +00001511 if (flags & BP_GDB)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001512 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
aliguoric0ce9982008-11-25 22:13:57 +00001513 else
Blue Swirl72cf2d42009-09-12 07:36:22 +00001514 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
aliguoria1d1bb32008-11-18 20:07:32 +00001515
1516 breakpoint_invalidate(env, pc);
1517
1518 if (breakpoint)
1519 *breakpoint = bp;
1520 return 0;
1521#else
1522 return -ENOSYS;
1523#endif
1524}
1525
1526/* Remove a specific breakpoint. */
1527int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1528{
1529#if defined(TARGET_HAS_ICE)
1530 CPUBreakpoint *bp;
1531
Blue Swirl72cf2d42009-09-12 07:36:22 +00001532 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
aliguoria1d1bb32008-11-18 20:07:32 +00001533 if (bp->pc == pc && bp->flags == flags) {
1534 cpu_breakpoint_remove_by_ref(env, bp);
bellard4c3a88a2003-07-26 12:06:08 +00001535 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +00001536 }
bellard4c3a88a2003-07-26 12:06:08 +00001537 }
aliguoria1d1bb32008-11-18 20:07:32 +00001538 return -ENOENT;
bellard4c3a88a2003-07-26 12:06:08 +00001539#else
aliguoria1d1bb32008-11-18 20:07:32 +00001540 return -ENOSYS;
bellard4c3a88a2003-07-26 12:06:08 +00001541#endif
1542}
1543
aliguoria1d1bb32008-11-18 20:07:32 +00001544/* Remove a specific breakpoint by reference. */
1545void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +00001546{
bellard1fddef42005-04-17 19:16:13 +00001547#if defined(TARGET_HAS_ICE)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001548 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
bellardd720b932004-04-25 17:57:43 +00001549
aliguoria1d1bb32008-11-18 20:07:32 +00001550 breakpoint_invalidate(env, breakpoint->pc);
1551
1552 qemu_free(breakpoint);
1553#endif
1554}
1555
1556/* Remove all matching breakpoints. */
1557void cpu_breakpoint_remove_all(CPUState *env, int mask)
1558{
1559#if defined(TARGET_HAS_ICE)
aliguoric0ce9982008-11-25 22:13:57 +00001560 CPUBreakpoint *bp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +00001561
Blue Swirl72cf2d42009-09-12 07:36:22 +00001562 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
aliguoria1d1bb32008-11-18 20:07:32 +00001563 if (bp->flags & mask)
1564 cpu_breakpoint_remove_by_ref(env, bp);
aliguoric0ce9982008-11-25 22:13:57 +00001565 }
bellard4c3a88a2003-07-26 12:06:08 +00001566#endif
1567}
1568
bellardc33a3462003-07-29 20:50:33 +00001569/* enable or disable single step mode. EXCP_DEBUG is returned by the
1570 CPU loop after each instruction */
1571void cpu_single_step(CPUState *env, int enabled)
1572{
bellard1fddef42005-04-17 19:16:13 +00001573#if defined(TARGET_HAS_ICE)
bellardc33a3462003-07-29 20:50:33 +00001574 if (env->singlestep_enabled != enabled) {
1575 env->singlestep_enabled = enabled;
aliguorie22a25c2009-03-12 20:12:48 +00001576 if (kvm_enabled())
1577 kvm_update_guest_debug(env, 0);
1578 else {
Stuart Bradyccbb4d42009-05-03 12:15:06 +01001579 /* must flush all the translated code to avoid inconsistencies */
aliguorie22a25c2009-03-12 20:12:48 +00001580 /* XXX: only flush what is necessary */
1581 tb_flush(env);
1582 }
bellardc33a3462003-07-29 20:50:33 +00001583 }
1584#endif
1585}
1586
bellard34865132003-10-05 14:28:56 +00001587/* enable or disable low levels log */
1588void cpu_set_log(int log_flags)
1589{
1590 loglevel = log_flags;
1591 if (loglevel && !logfile) {
pbrook11fcfab2007-07-01 18:21:11 +00001592 logfile = fopen(logfilename, log_append ? "a" : "w");
bellard34865132003-10-05 14:28:56 +00001593 if (!logfile) {
1594 perror(logfilename);
1595 _exit(1);
1596 }
bellard9fa3e852004-01-04 18:06:42 +00001597#if !defined(CONFIG_SOFTMMU)
1598 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1599 {
blueswir1b55266b2008-09-20 08:07:15 +00001600 static char logfile_buf[4096];
bellard9fa3e852004-01-04 18:06:42 +00001601 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1602 }
Filip Navarabf65f532009-07-27 10:02:04 -05001603#elif !defined(_WIN32)
1604 /* Win32 doesn't support line-buffering and requires size >= 2 */
bellard34865132003-10-05 14:28:56 +00001605 setvbuf(logfile, NULL, _IOLBF, 0);
bellard9fa3e852004-01-04 18:06:42 +00001606#endif
pbrooke735b912007-06-30 13:53:24 +00001607 log_append = 1;
1608 }
1609 if (!loglevel && logfile) {
1610 fclose(logfile);
1611 logfile = NULL;
bellard34865132003-10-05 14:28:56 +00001612 }
1613}
1614
1615void cpu_set_log_filename(const char *filename)
1616{
1617 logfilename = strdup(filename);
pbrooke735b912007-06-30 13:53:24 +00001618 if (logfile) {
1619 fclose(logfile);
1620 logfile = NULL;
1621 }
1622 cpu_set_log(loglevel);
bellard34865132003-10-05 14:28:56 +00001623}
bellardc33a3462003-07-29 20:50:33 +00001624
aurel323098dba2009-03-07 21:28:24 +00001625static void cpu_unlink_tb(CPUState *env)
bellardea041c02003-06-25 16:16:50 +00001626{
pbrookd5975362008-06-07 20:50:51 +00001627 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1628 problem and hope the cpu will stop of its own accord. For userspace
1629 emulation this often isn't actually as bad as it sounds. Often
1630 signals are used primarily to interrupt blocking syscalls. */
aurel323098dba2009-03-07 21:28:24 +00001631 TranslationBlock *tb;
Anthony Liguoric227f092009-10-01 16:12:16 -05001632 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
aurel323098dba2009-03-07 21:28:24 +00001633
Riku Voipiocab1b4b2010-01-20 12:56:27 +02001634 spin_lock(&interrupt_lock);
aurel323098dba2009-03-07 21:28:24 +00001635 tb = env->current_tb;
1636 /* if the cpu is currently executing code, we must unlink it and
1637 all the potentially executing TB */
Riku Voipiof76cfe52009-12-04 15:16:30 +02001638 if (tb) {
aurel323098dba2009-03-07 21:28:24 +00001639 env->current_tb = NULL;
1640 tb_reset_jump_recursive(tb);
aurel323098dba2009-03-07 21:28:24 +00001641 }
Riku Voipiocab1b4b2010-01-20 12:56:27 +02001642 spin_unlock(&interrupt_lock);
aurel323098dba2009-03-07 21:28:24 +00001643}
1644
Jan Kiszka97ffbd82011-04-13 01:32:56 +02001645#ifndef CONFIG_USER_ONLY
aurel323098dba2009-03-07 21:28:24 +00001646/* mask must never be zero, except for A20 change call */
Jan Kiszkaec6959d2011-04-13 01:32:56 +02001647static void tcg_handle_interrupt(CPUState *env, int mask)
aurel323098dba2009-03-07 21:28:24 +00001648{
1649 int old_mask;
1650
1651 old_mask = env->interrupt_request;
1652 env->interrupt_request |= mask;
1653
aliguori8edac962009-04-24 18:03:45 +00001654 /*
1655 * If called from iothread context, wake the target cpu in
1656 * case its halted.
1657 */
Jan Kiszkab7680cb2011-03-12 17:43:51 +01001658 if (!qemu_cpu_is_self(env)) {
aliguori8edac962009-04-24 18:03:45 +00001659 qemu_cpu_kick(env);
1660 return;
1661 }
aliguori8edac962009-04-24 18:03:45 +00001662
pbrook2e70f6e2008-06-29 01:03:05 +00001663 if (use_icount) {
pbrook266910c2008-07-09 15:31:50 +00001664 env->icount_decr.u16.high = 0xffff;
pbrook2e70f6e2008-06-29 01:03:05 +00001665 if (!can_do_io(env)
aurel32be214e62009-03-06 21:48:00 +00001666 && (mask & ~old_mask) != 0) {
pbrook2e70f6e2008-06-29 01:03:05 +00001667 cpu_abort(env, "Raised interrupt while not in I/O function");
1668 }
pbrook2e70f6e2008-06-29 01:03:05 +00001669 } else {
aurel323098dba2009-03-07 21:28:24 +00001670 cpu_unlink_tb(env);
bellardea041c02003-06-25 16:16:50 +00001671 }
1672}
1673
Jan Kiszkaec6959d2011-04-13 01:32:56 +02001674CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1675
Jan Kiszka97ffbd82011-04-13 01:32:56 +02001676#else /* CONFIG_USER_ONLY */
1677
1678void cpu_interrupt(CPUState *env, int mask)
1679{
1680 env->interrupt_request |= mask;
1681 cpu_unlink_tb(env);
1682}
1683#endif /* CONFIG_USER_ONLY */
1684
bellardb54ad042004-05-20 13:42:52 +00001685void cpu_reset_interrupt(CPUState *env, int mask)
1686{
1687 env->interrupt_request &= ~mask;
1688}
1689
aurel323098dba2009-03-07 21:28:24 +00001690void cpu_exit(CPUState *env)
1691{
1692 env->exit_request = 1;
1693 cpu_unlink_tb(env);
1694}
1695
blueswir1c7cd6a32008-10-02 18:27:46 +00001696const CPULogItem cpu_log_items[] = {
ths5fafdf22007-09-16 21:08:06 +00001697 { CPU_LOG_TB_OUT_ASM, "out_asm",
bellardf193c792004-03-21 17:06:25 +00001698 "show generated host assembly code for each compiled TB" },
1699 { CPU_LOG_TB_IN_ASM, "in_asm",
1700 "show target assembly code for each compiled TB" },
ths5fafdf22007-09-16 21:08:06 +00001701 { CPU_LOG_TB_OP, "op",
bellard57fec1f2008-02-01 10:50:11 +00001702 "show micro ops for each compiled TB" },
bellardf193c792004-03-21 17:06:25 +00001703 { CPU_LOG_TB_OP_OPT, "op_opt",
blueswir1e01a1152008-03-14 17:37:11 +00001704 "show micro ops "
1705#ifdef TARGET_I386
1706 "before eflags optimization and "
bellardf193c792004-03-21 17:06:25 +00001707#endif
blueswir1e01a1152008-03-14 17:37:11 +00001708 "after liveness analysis" },
bellardf193c792004-03-21 17:06:25 +00001709 { CPU_LOG_INT, "int",
1710 "show interrupts/exceptions in short format" },
1711 { CPU_LOG_EXEC, "exec",
1712 "show trace before each executed TB (lots of logs)" },
bellard9fddaa02004-05-21 12:59:32 +00001713 { CPU_LOG_TB_CPU, "cpu",
thse91c8a72007-06-03 13:35:16 +00001714 "show CPU state before block translation" },
bellardf193c792004-03-21 17:06:25 +00001715#ifdef TARGET_I386
1716 { CPU_LOG_PCALL, "pcall",
1717 "show protected mode far calls/returns/exceptions" },
aliguorieca1bdf2009-01-26 19:54:31 +00001718 { CPU_LOG_RESET, "cpu_reset",
1719 "show CPU state before CPU resets" },
bellardf193c792004-03-21 17:06:25 +00001720#endif
bellard8e3a9fd2004-10-09 17:32:58 +00001721#ifdef DEBUG_IOPORT
bellardfd872592004-05-12 19:11:15 +00001722 { CPU_LOG_IOPORT, "ioport",
1723 "show all i/o ports accesses" },
bellard8e3a9fd2004-10-09 17:32:58 +00001724#endif
bellardf193c792004-03-21 17:06:25 +00001725 { 0, NULL, NULL },
1726};
1727
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001728#ifndef CONFIG_USER_ONLY
1729static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1730 = QLIST_HEAD_INITIALIZER(memory_client_list);
1731
1732static void cpu_notify_set_memory(target_phys_addr_t start_addr,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001733 ram_addr_t size,
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03001734 ram_addr_t phys_offset,
1735 bool log_dirty)
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001736{
1737 CPUPhysMemoryClient *client;
1738 QLIST_FOREACH(client, &memory_client_list, list) {
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03001739 client->set_memory(client, start_addr, size, phys_offset, log_dirty);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001740 }
1741}
1742
1743static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001744 target_phys_addr_t end)
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001745{
1746 CPUPhysMemoryClient *client;
1747 QLIST_FOREACH(client, &memory_client_list, list) {
1748 int r = client->sync_dirty_bitmap(client, start, end);
1749 if (r < 0)
1750 return r;
1751 }
1752 return 0;
1753}
1754
1755static int cpu_notify_migration_log(int enable)
1756{
1757 CPUPhysMemoryClient *client;
1758 QLIST_FOREACH(client, &memory_client_list, list) {
1759 int r = client->migration_log(client, enable);
1760 if (r < 0)
1761 return r;
1762 }
1763 return 0;
1764}
1765
Alex Williamson2173a752011-05-03 12:36:58 -06001766struct last_map {
1767 target_phys_addr_t start_addr;
1768 ram_addr_t size;
1769 ram_addr_t phys_offset;
1770};
1771
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001772/* The l1_phys_map provides the upper P_L1_BITs of the guest physical
1773 * address. Each intermediate table provides the next L2_BITs of guest
1774 * physical address space. The number of levels vary based on host and
1775 * guest configuration, making it efficient to build the final guest
1776 * physical address by seeding the L1 offset and shifting and adding in
1777 * each L2 offset as we recurse through them. */
Alex Williamson2173a752011-05-03 12:36:58 -06001778static void phys_page_for_each_1(CPUPhysMemoryClient *client, int level,
1779 void **lp, target_phys_addr_t addr,
1780 struct last_map *map)
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001781{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001782 int i;
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001783
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001784 if (*lp == NULL) {
1785 return;
1786 }
1787 if (level == 0) {
1788 PhysPageDesc *pd = *lp;
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001789 addr <<= L2_BITS + TARGET_PAGE_BITS;
Paul Brook7296aba2010-03-14 14:58:46 +00001790 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001791 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
Alex Williamson2173a752011-05-03 12:36:58 -06001792 target_phys_addr_t start_addr = addr | i << TARGET_PAGE_BITS;
1793
1794 if (map->size &&
1795 start_addr == map->start_addr + map->size &&
1796 pd[i].phys_offset == map->phys_offset + map->size) {
1797
1798 map->size += TARGET_PAGE_SIZE;
1799 continue;
1800 } else if (map->size) {
1801 client->set_memory(client, map->start_addr,
1802 map->size, map->phys_offset, false);
1803 }
1804
1805 map->start_addr = start_addr;
1806 map->size = TARGET_PAGE_SIZE;
1807 map->phys_offset = pd[i].phys_offset;
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001808 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001809 }
1810 } else {
1811 void **pp = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +00001812 for (i = 0; i < L2_SIZE; ++i) {
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001813 phys_page_for_each_1(client, level - 1, pp + i,
Alex Williamson2173a752011-05-03 12:36:58 -06001814 (addr << L2_BITS) | i, map);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001815 }
1816 }
1817}
1818
1819static void phys_page_for_each(CPUPhysMemoryClient *client)
1820{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001821 int i;
Alex Williamson2173a752011-05-03 12:36:58 -06001822 struct last_map map = { };
1823
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001824 for (i = 0; i < P_L1_SIZE; ++i) {
1825 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
Alex Williamson2173a752011-05-03 12:36:58 -06001826 l1_phys_map + i, i, &map);
1827 }
1828 if (map.size) {
1829 client->set_memory(client, map.start_addr, map.size, map.phys_offset,
1830 false);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001831 }
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001832}
1833
1834void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1835{
1836 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1837 phys_page_for_each(client);
1838}
1839
1840void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1841{
1842 QLIST_REMOVE(client, list);
1843}
1844#endif
1845
bellardf193c792004-03-21 17:06:25 +00001846static int cmp1(const char *s1, int n, const char *s2)
1847{
1848 if (strlen(s2) != n)
1849 return 0;
1850 return memcmp(s1, s2, n) == 0;
1851}
ths3b46e622007-09-17 08:09:54 +00001852
bellardf193c792004-03-21 17:06:25 +00001853/* takes a comma separated list of log masks. Return 0 if error. */
1854int cpu_str_to_log_mask(const char *str)
1855{
blueswir1c7cd6a32008-10-02 18:27:46 +00001856 const CPULogItem *item;
bellardf193c792004-03-21 17:06:25 +00001857 int mask;
1858 const char *p, *p1;
1859
1860 p = str;
1861 mask = 0;
1862 for(;;) {
1863 p1 = strchr(p, ',');
1864 if (!p1)
1865 p1 = p + strlen(p);
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001866 if(cmp1(p,p1-p,"all")) {
1867 for(item = cpu_log_items; item->mask != 0; item++) {
1868 mask |= item->mask;
1869 }
1870 } else {
1871 for(item = cpu_log_items; item->mask != 0; item++) {
1872 if (cmp1(p, p1 - p, item->name))
1873 goto found;
1874 }
1875 return 0;
bellardf193c792004-03-21 17:06:25 +00001876 }
bellardf193c792004-03-21 17:06:25 +00001877 found:
1878 mask |= item->mask;
1879 if (*p1 != ',')
1880 break;
1881 p = p1 + 1;
1882 }
1883 return mask;
1884}
bellardea041c02003-06-25 16:16:50 +00001885
bellard75012672003-06-21 13:11:07 +00001886void cpu_abort(CPUState *env, const char *fmt, ...)
1887{
1888 va_list ap;
pbrook493ae1f2007-11-23 16:53:59 +00001889 va_list ap2;
bellard75012672003-06-21 13:11:07 +00001890
1891 va_start(ap, fmt);
pbrook493ae1f2007-11-23 16:53:59 +00001892 va_copy(ap2, ap);
bellard75012672003-06-21 13:11:07 +00001893 fprintf(stderr, "qemu: fatal: ");
1894 vfprintf(stderr, fmt, ap);
1895 fprintf(stderr, "\n");
1896#ifdef TARGET_I386
bellard7fe48482004-10-09 18:08:01 +00001897 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1898#else
1899 cpu_dump_state(env, stderr, fprintf, 0);
bellard75012672003-06-21 13:11:07 +00001900#endif
aliguori93fcfe32009-01-15 22:34:14 +00001901 if (qemu_log_enabled()) {
1902 qemu_log("qemu: fatal: ");
1903 qemu_log_vprintf(fmt, ap2);
1904 qemu_log("\n");
j_mayerf9373292007-09-29 12:18:20 +00001905#ifdef TARGET_I386
aliguori93fcfe32009-01-15 22:34:14 +00001906 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
j_mayerf9373292007-09-29 12:18:20 +00001907#else
aliguori93fcfe32009-01-15 22:34:14 +00001908 log_cpu_state(env, 0);
j_mayerf9373292007-09-29 12:18:20 +00001909#endif
aliguori31b1a7b2009-01-15 22:35:09 +00001910 qemu_log_flush();
aliguori93fcfe32009-01-15 22:34:14 +00001911 qemu_log_close();
balrog924edca2007-06-10 14:07:13 +00001912 }
pbrook493ae1f2007-11-23 16:53:59 +00001913 va_end(ap2);
j_mayerf9373292007-09-29 12:18:20 +00001914 va_end(ap);
Riku Voipiofd052bf2010-01-25 14:30:49 +02001915#if defined(CONFIG_USER_ONLY)
1916 {
1917 struct sigaction act;
1918 sigfillset(&act.sa_mask);
1919 act.sa_handler = SIG_DFL;
1920 sigaction(SIGABRT, &act, NULL);
1921 }
1922#endif
bellard75012672003-06-21 13:11:07 +00001923 abort();
1924}
1925
thsc5be9f02007-02-28 20:20:53 +00001926CPUState *cpu_copy(CPUState *env)
1927{
ths01ba9812007-12-09 02:22:57 +00001928 CPUState *new_env = cpu_init(env->cpu_model_str);
thsc5be9f02007-02-28 20:20:53 +00001929 CPUState *next_cpu = new_env->next_cpu;
1930 int cpu_index = new_env->cpu_index;
aliguori5a38f082009-01-15 20:16:51 +00001931#if defined(TARGET_HAS_ICE)
1932 CPUBreakpoint *bp;
1933 CPUWatchpoint *wp;
1934#endif
1935
thsc5be9f02007-02-28 20:20:53 +00001936 memcpy(new_env, env, sizeof(CPUState));
aliguori5a38f082009-01-15 20:16:51 +00001937
1938 /* Preserve chaining and index. */
thsc5be9f02007-02-28 20:20:53 +00001939 new_env->next_cpu = next_cpu;
1940 new_env->cpu_index = cpu_index;
aliguori5a38f082009-01-15 20:16:51 +00001941
1942 /* Clone all break/watchpoints.
1943 Note: Once we support ptrace with hw-debug register access, make sure
1944 BP_CPU break/watchpoints are handled correctly on clone. */
Blue Swirl72cf2d42009-09-12 07:36:22 +00001945 QTAILQ_INIT(&env->breakpoints);
1946 QTAILQ_INIT(&env->watchpoints);
aliguori5a38f082009-01-15 20:16:51 +00001947#if defined(TARGET_HAS_ICE)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001948 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
aliguori5a38f082009-01-15 20:16:51 +00001949 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1950 }
Blue Swirl72cf2d42009-09-12 07:36:22 +00001951 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguori5a38f082009-01-15 20:16:51 +00001952 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1953 wp->flags, NULL);
1954 }
1955#endif
1956
thsc5be9f02007-02-28 20:20:53 +00001957 return new_env;
1958}
1959
bellard01243112004-01-04 15:48:17 +00001960#if !defined(CONFIG_USER_ONLY)
1961
edgar_igl5c751e92008-05-06 08:44:21 +00001962static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1963{
1964 unsigned int i;
1965
1966 /* Discard jump cache entries for any tb which might potentially
1967 overlap the flushed page. */
1968 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1969 memset (&env->tb_jmp_cache[i], 0,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001970 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
edgar_igl5c751e92008-05-06 08:44:21 +00001971
1972 i = tb_jmp_cache_hash_page(addr);
1973 memset (&env->tb_jmp_cache[i], 0,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001974 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
edgar_igl5c751e92008-05-06 08:44:21 +00001975}
1976
Igor Kovalenko08738982009-07-12 02:15:40 +04001977static CPUTLBEntry s_cputlb_empty_entry = {
1978 .addr_read = -1,
1979 .addr_write = -1,
1980 .addr_code = -1,
1981 .addend = -1,
1982};
1983
bellardee8b7022004-02-03 23:35:10 +00001984/* NOTE: if flush_global is true, also flush global entries (not
1985 implemented yet) */
1986void tlb_flush(CPUState *env, int flush_global)
bellard33417e72003-08-10 21:47:01 +00001987{
bellard33417e72003-08-10 21:47:01 +00001988 int i;
bellard01243112004-01-04 15:48:17 +00001989
bellard9fa3e852004-01-04 18:06:42 +00001990#if defined(DEBUG_TLB)
1991 printf("tlb_flush:\n");
1992#endif
bellard01243112004-01-04 15:48:17 +00001993 /* must reset current TB so that interrupts cannot modify the
1994 links while we are modifying them */
1995 env->current_tb = NULL;
1996
bellard33417e72003-08-10 21:47:01 +00001997 for(i = 0; i < CPU_TLB_SIZE; i++) {
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09001998 int mmu_idx;
1999 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
Igor Kovalenko08738982009-07-12 02:15:40 +04002000 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002001 }
bellard33417e72003-08-10 21:47:01 +00002002 }
bellard9fa3e852004-01-04 18:06:42 +00002003
bellard8a40a182005-11-20 10:35:40 +00002004 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
bellard9fa3e852004-01-04 18:06:42 +00002005
Paul Brookd4c430a2010-03-17 02:14:28 +00002006 env->tlb_flush_addr = -1;
2007 env->tlb_flush_mask = 0;
bellarde3db7222005-01-26 22:00:47 +00002008 tlb_flush_count++;
bellard33417e72003-08-10 21:47:01 +00002009}
2010
bellard274da6b2004-05-20 21:56:27 +00002011static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
bellard61382a52003-10-27 21:22:23 +00002012{
ths5fafdf22007-09-16 21:08:06 +00002013 if (addr == (tlb_entry->addr_read &
bellard84b7b8e2005-11-28 21:19:04 +00002014 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
ths5fafdf22007-09-16 21:08:06 +00002015 addr == (tlb_entry->addr_write &
bellard84b7b8e2005-11-28 21:19:04 +00002016 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
ths5fafdf22007-09-16 21:08:06 +00002017 addr == (tlb_entry->addr_code &
bellard84b7b8e2005-11-28 21:19:04 +00002018 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
Igor Kovalenko08738982009-07-12 02:15:40 +04002019 *tlb_entry = s_cputlb_empty_entry;
bellard84b7b8e2005-11-28 21:19:04 +00002020 }
bellard61382a52003-10-27 21:22:23 +00002021}
2022
bellard2e126692004-04-25 21:28:44 +00002023void tlb_flush_page(CPUState *env, target_ulong addr)
bellard33417e72003-08-10 21:47:01 +00002024{
bellard8a40a182005-11-20 10:35:40 +00002025 int i;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002026 int mmu_idx;
bellard01243112004-01-04 15:48:17 +00002027
bellard9fa3e852004-01-04 18:06:42 +00002028#if defined(DEBUG_TLB)
bellard108c49b2005-07-24 12:55:09 +00002029 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
bellard9fa3e852004-01-04 18:06:42 +00002030#endif
Paul Brookd4c430a2010-03-17 02:14:28 +00002031 /* Check if we need to flush due to large pages. */
2032 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
2033#if defined(DEBUG_TLB)
2034 printf("tlb_flush_page: forced full flush ("
2035 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
2036 env->tlb_flush_addr, env->tlb_flush_mask);
2037#endif
2038 tlb_flush(env, 1);
2039 return;
2040 }
bellard01243112004-01-04 15:48:17 +00002041 /* must reset current TB so that interrupts cannot modify the
2042 links while we are modifying them */
2043 env->current_tb = NULL;
bellard33417e72003-08-10 21:47:01 +00002044
bellard61382a52003-10-27 21:22:23 +00002045 addr &= TARGET_PAGE_MASK;
bellard33417e72003-08-10 21:47:01 +00002046 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002047 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2048 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
bellard01243112004-01-04 15:48:17 +00002049
edgar_igl5c751e92008-05-06 08:44:21 +00002050 tlb_flush_jmp_cache(env, addr);
bellard9fa3e852004-01-04 18:06:42 +00002051}
2052
bellard9fa3e852004-01-04 18:06:42 +00002053/* update the TLBs so that writes to code in the virtual page 'addr'
2054 can be detected */
Anthony Liguoric227f092009-10-01 16:12:16 -05002055static void tlb_protect_code(ram_addr_t ram_addr)
bellard61382a52003-10-27 21:22:23 +00002056{
ths5fafdf22007-09-16 21:08:06 +00002057 cpu_physical_memory_reset_dirty(ram_addr,
bellard6a00d602005-11-21 23:25:50 +00002058 ram_addr + TARGET_PAGE_SIZE,
2059 CODE_DIRTY_FLAG);
bellard9fa3e852004-01-04 18:06:42 +00002060}
2061
bellard9fa3e852004-01-04 18:06:42 +00002062/* update the TLB so that writes in physical page 'phys_addr' are no longer
bellard3a7d9292005-08-21 09:26:42 +00002063 tested for self modifying code */
Anthony Liguoric227f092009-10-01 16:12:16 -05002064static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
bellard3a7d9292005-08-21 09:26:42 +00002065 target_ulong vaddr)
bellard9fa3e852004-01-04 18:06:42 +00002066{
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09002067 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
bellard1ccde1c2004-02-06 19:46:14 +00002068}
2069
ths5fafdf22007-09-16 21:08:06 +00002070static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
bellard1ccde1c2004-02-06 19:46:14 +00002071 unsigned long start, unsigned long length)
2072{
2073 unsigned long addr;
bellard84b7b8e2005-11-28 21:19:04 +00002074 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2075 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
bellard1ccde1c2004-02-06 19:46:14 +00002076 if ((addr - start) < length) {
pbrook0f459d12008-06-09 00:20:13 +00002077 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
bellard1ccde1c2004-02-06 19:46:14 +00002078 }
2079 }
2080}
2081
pbrook5579c7f2009-04-11 14:47:08 +00002082/* Note: start and end must be within the same ram block. */
Anthony Liguoric227f092009-10-01 16:12:16 -05002083void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
bellard0a962c02005-02-10 22:00:27 +00002084 int dirty_flags)
bellard1ccde1c2004-02-06 19:46:14 +00002085{
2086 CPUState *env;
bellard4f2ac232004-04-26 19:44:02 +00002087 unsigned long length, start1;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09002088 int i;
bellard1ccde1c2004-02-06 19:46:14 +00002089
2090 start &= TARGET_PAGE_MASK;
2091 end = TARGET_PAGE_ALIGN(end);
2092
2093 length = end - start;
2094 if (length == 0)
2095 return;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09002096 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00002097
bellard1ccde1c2004-02-06 19:46:14 +00002098 /* we modify the TLB cache so that the dirty bit will be set again
2099 when accessing the range */
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02002100 start1 = (unsigned long)qemu_safe_ram_ptr(start);
Stefan Weila57d23e2011-04-30 22:49:26 +02002101 /* Check that we don't span multiple blocks - this breaks the
pbrook5579c7f2009-04-11 14:47:08 +00002102 address comparisons below. */
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02002103 if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
pbrook5579c7f2009-04-11 14:47:08 +00002104 != (end - 1) - start) {
2105 abort();
2106 }
2107
bellard6a00d602005-11-21 23:25:50 +00002108 for(env = first_cpu; env != NULL; env = env->next_cpu) {
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002109 int mmu_idx;
2110 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2111 for(i = 0; i < CPU_TLB_SIZE; i++)
2112 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2113 start1, length);
2114 }
bellard6a00d602005-11-21 23:25:50 +00002115 }
bellard1ccde1c2004-02-06 19:46:14 +00002116}
2117
aliguori74576192008-10-06 14:02:03 +00002118int cpu_physical_memory_set_dirty_tracking(int enable)
2119{
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002120 int ret = 0;
aliguori74576192008-10-06 14:02:03 +00002121 in_migration = enable;
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002122 ret = cpu_notify_migration_log(!!enable);
2123 return ret;
aliguori74576192008-10-06 14:02:03 +00002124}
2125
2126int cpu_physical_memory_get_dirty_tracking(void)
2127{
2128 return in_migration;
2129}
2130
Anthony Liguoric227f092009-10-01 16:12:16 -05002131int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2132 target_phys_addr_t end_addr)
aliguori2bec46d2008-11-24 20:21:41 +00002133{
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +02002134 int ret;
Jan Kiszka151f7742009-05-01 20:52:47 +02002135
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002136 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
Jan Kiszka151f7742009-05-01 20:52:47 +02002137 return ret;
aliguori2bec46d2008-11-24 20:21:41 +00002138}
2139
Anthony PERARDe5896b12011-02-07 12:19:23 +01002140int cpu_physical_log_start(target_phys_addr_t start_addr,
2141 ram_addr_t size)
2142{
2143 CPUPhysMemoryClient *client;
2144 QLIST_FOREACH(client, &memory_client_list, list) {
2145 if (client->log_start) {
2146 int r = client->log_start(client, start_addr, size);
2147 if (r < 0) {
2148 return r;
2149 }
2150 }
2151 }
2152 return 0;
2153}
2154
2155int cpu_physical_log_stop(target_phys_addr_t start_addr,
2156 ram_addr_t size)
2157{
2158 CPUPhysMemoryClient *client;
2159 QLIST_FOREACH(client, &memory_client_list, list) {
2160 if (client->log_stop) {
2161 int r = client->log_stop(client, start_addr, size);
2162 if (r < 0) {
2163 return r;
2164 }
2165 }
2166 }
2167 return 0;
2168}
2169
bellard3a7d9292005-08-21 09:26:42 +00002170static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2171{
Anthony Liguoric227f092009-10-01 16:12:16 -05002172 ram_addr_t ram_addr;
pbrook5579c7f2009-04-11 14:47:08 +00002173 void *p;
bellard3a7d9292005-08-21 09:26:42 +00002174
bellard84b7b8e2005-11-28 21:19:04 +00002175 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
pbrook5579c7f2009-04-11 14:47:08 +00002176 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2177 + tlb_entry->addend);
Marcelo Tosattie8902612010-10-11 15:31:19 -03002178 ram_addr = qemu_ram_addr_from_host_nofail(p);
bellard3a7d9292005-08-21 09:26:42 +00002179 if (!cpu_physical_memory_is_dirty(ram_addr)) {
pbrook0f459d12008-06-09 00:20:13 +00002180 tlb_entry->addr_write |= TLB_NOTDIRTY;
bellard3a7d9292005-08-21 09:26:42 +00002181 }
2182 }
2183}
2184
2185/* update the TLB according to the current state of the dirty bits */
2186void cpu_tlb_update_dirty(CPUState *env)
2187{
2188 int i;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002189 int mmu_idx;
2190 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2191 for(i = 0; i < CPU_TLB_SIZE; i++)
2192 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2193 }
bellard3a7d9292005-08-21 09:26:42 +00002194}
2195
pbrook0f459d12008-06-09 00:20:13 +00002196static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
bellard1ccde1c2004-02-06 19:46:14 +00002197{
pbrook0f459d12008-06-09 00:20:13 +00002198 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2199 tlb_entry->addr_write = vaddr;
bellard1ccde1c2004-02-06 19:46:14 +00002200}
2201
pbrook0f459d12008-06-09 00:20:13 +00002202/* update the TLB corresponding to virtual page vaddr
2203 so that it is no longer dirty */
2204static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
bellard1ccde1c2004-02-06 19:46:14 +00002205{
bellard1ccde1c2004-02-06 19:46:14 +00002206 int i;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002207 int mmu_idx;
bellard1ccde1c2004-02-06 19:46:14 +00002208
pbrook0f459d12008-06-09 00:20:13 +00002209 vaddr &= TARGET_PAGE_MASK;
bellard1ccde1c2004-02-06 19:46:14 +00002210 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002211 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2212 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
bellard9fa3e852004-01-04 18:06:42 +00002213}
2214
Paul Brookd4c430a2010-03-17 02:14:28 +00002215/* Our TLB does not support large pages, so remember the area covered by
2216 large pages and trigger a full TLB flush if these are invalidated. */
2217static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2218 target_ulong size)
2219{
2220 target_ulong mask = ~(size - 1);
2221
2222 if (env->tlb_flush_addr == (target_ulong)-1) {
2223 env->tlb_flush_addr = vaddr & mask;
2224 env->tlb_flush_mask = mask;
2225 return;
2226 }
2227 /* Extend the existing region to include the new page.
2228 This is a compromise between unnecessary flushes and the cost
2229 of maintaining a full variable size TLB. */
2230 mask &= env->tlb_flush_mask;
2231 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2232 mask <<= 1;
2233 }
2234 env->tlb_flush_addr &= mask;
2235 env->tlb_flush_mask = mask;
2236}
2237
2238/* Add a new TLB entry. At most one entry for a given virtual address
2239 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2240 supplied size is only used by tlb_flush_page. */
2241void tlb_set_page(CPUState *env, target_ulong vaddr,
2242 target_phys_addr_t paddr, int prot,
2243 int mmu_idx, target_ulong size)
bellard9fa3e852004-01-04 18:06:42 +00002244{
bellard92e873b2004-05-21 14:52:29 +00002245 PhysPageDesc *p;
bellard4f2ac232004-04-26 19:44:02 +00002246 unsigned long pd;
bellard9fa3e852004-01-04 18:06:42 +00002247 unsigned int index;
bellard4f2ac232004-04-26 19:44:02 +00002248 target_ulong address;
pbrook0f459d12008-06-09 00:20:13 +00002249 target_ulong code_address;
Paul Brook355b1942010-04-05 00:28:53 +01002250 unsigned long addend;
bellard84b7b8e2005-11-28 21:19:04 +00002251 CPUTLBEntry *te;
aliguoria1d1bb32008-11-18 20:07:32 +00002252 CPUWatchpoint *wp;
Anthony Liguoric227f092009-10-01 16:12:16 -05002253 target_phys_addr_t iotlb;
bellard9fa3e852004-01-04 18:06:42 +00002254
Paul Brookd4c430a2010-03-17 02:14:28 +00002255 assert(size >= TARGET_PAGE_SIZE);
2256 if (size != TARGET_PAGE_SIZE) {
2257 tlb_add_large_page(env, vaddr, size);
2258 }
bellard92e873b2004-05-21 14:52:29 +00002259 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
bellard9fa3e852004-01-04 18:06:42 +00002260 if (!p) {
2261 pd = IO_MEM_UNASSIGNED;
bellard9fa3e852004-01-04 18:06:42 +00002262 } else {
2263 pd = p->phys_offset;
bellard9fa3e852004-01-04 18:06:42 +00002264 }
2265#if defined(DEBUG_TLB)
Stefan Weil7fd3f492010-09-30 22:39:51 +02002266 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2267 " prot=%x idx=%d pd=0x%08lx\n",
2268 vaddr, paddr, prot, mmu_idx, pd);
bellard9fa3e852004-01-04 18:06:42 +00002269#endif
2270
pbrook0f459d12008-06-09 00:20:13 +00002271 address = vaddr;
2272 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2273 /* IO memory case (romd handled later) */
2274 address |= TLB_MMIO;
2275 }
pbrook5579c7f2009-04-11 14:47:08 +00002276 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
pbrook0f459d12008-06-09 00:20:13 +00002277 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2278 /* Normal RAM. */
2279 iotlb = pd & TARGET_PAGE_MASK;
2280 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2281 iotlb |= IO_MEM_NOTDIRTY;
2282 else
2283 iotlb |= IO_MEM_ROM;
2284 } else {
Stuart Bradyccbb4d42009-05-03 12:15:06 +01002285 /* IO handlers are currently passed a physical address.
pbrook0f459d12008-06-09 00:20:13 +00002286 It would be nice to pass an offset from the base address
2287 of that region. This would avoid having to special case RAM,
2288 and avoid full address decoding in every device.
2289 We can't use the high bits of pd for this because
2290 IO_MEM_ROMD uses these as a ram address. */
pbrook8da3ff12008-12-01 18:59:50 +00002291 iotlb = (pd & ~TARGET_PAGE_MASK);
2292 if (p) {
pbrook8da3ff12008-12-01 18:59:50 +00002293 iotlb += p->region_offset;
2294 } else {
2295 iotlb += paddr;
2296 }
pbrook0f459d12008-06-09 00:20:13 +00002297 }
pbrook6658ffb2007-03-16 23:58:11 +00002298
pbrook0f459d12008-06-09 00:20:13 +00002299 code_address = address;
2300 /* Make accesses to pages with watchpoints go via the
2301 watchpoint trap routines. */
Blue Swirl72cf2d42009-09-12 07:36:22 +00002302 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguoria1d1bb32008-11-18 20:07:32 +00002303 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
Jun Koibf298f82010-05-06 14:36:59 +09002304 /* Avoid trapping reads of pages with a write breakpoint. */
2305 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2306 iotlb = io_mem_watch + paddr;
2307 address |= TLB_MMIO;
2308 break;
2309 }
pbrook6658ffb2007-03-16 23:58:11 +00002310 }
pbrook0f459d12008-06-09 00:20:13 +00002311 }
balrogd79acba2007-06-26 20:01:13 +00002312
pbrook0f459d12008-06-09 00:20:13 +00002313 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2314 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2315 te = &env->tlb_table[mmu_idx][index];
2316 te->addend = addend - vaddr;
2317 if (prot & PAGE_READ) {
2318 te->addr_read = address;
2319 } else {
2320 te->addr_read = -1;
2321 }
edgar_igl5c751e92008-05-06 08:44:21 +00002322
pbrook0f459d12008-06-09 00:20:13 +00002323 if (prot & PAGE_EXEC) {
2324 te->addr_code = code_address;
2325 } else {
2326 te->addr_code = -1;
2327 }
2328 if (prot & PAGE_WRITE) {
2329 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2330 (pd & IO_MEM_ROMD)) {
2331 /* Write access calls the I/O callback. */
2332 te->addr_write = address | TLB_MMIO;
2333 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2334 !cpu_physical_memory_is_dirty(pd)) {
2335 te->addr_write = address | TLB_NOTDIRTY;
bellard84b7b8e2005-11-28 21:19:04 +00002336 } else {
pbrook0f459d12008-06-09 00:20:13 +00002337 te->addr_write = address;
bellard9fa3e852004-01-04 18:06:42 +00002338 }
pbrook0f459d12008-06-09 00:20:13 +00002339 } else {
2340 te->addr_write = -1;
bellard9fa3e852004-01-04 18:06:42 +00002341 }
bellard9fa3e852004-01-04 18:06:42 +00002342}
2343
bellard01243112004-01-04 15:48:17 +00002344#else
2345
bellardee8b7022004-02-03 23:35:10 +00002346void tlb_flush(CPUState *env, int flush_global)
bellard01243112004-01-04 15:48:17 +00002347{
2348}
2349
bellard2e126692004-04-25 21:28:44 +00002350void tlb_flush_page(CPUState *env, target_ulong addr)
bellard01243112004-01-04 15:48:17 +00002351{
2352}
2353
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002354/*
2355 * Walks guest process memory "regions" one by one
2356 * and calls callback function 'fn' for each region.
2357 */
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002358
2359struct walk_memory_regions_data
bellard9fa3e852004-01-04 18:06:42 +00002360{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002361 walk_memory_regions_fn fn;
2362 void *priv;
2363 unsigned long start;
2364 int prot;
2365};
bellard9fa3e852004-01-04 18:06:42 +00002366
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002367static int walk_memory_regions_end(struct walk_memory_regions_data *data,
Paul Brookb480d9b2010-03-12 23:23:29 +00002368 abi_ulong end, int new_prot)
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002369{
2370 if (data->start != -1ul) {
2371 int rc = data->fn(data->priv, data->start, end, data->prot);
2372 if (rc != 0) {
2373 return rc;
bellard9fa3e852004-01-04 18:06:42 +00002374 }
bellard33417e72003-08-10 21:47:01 +00002375 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002376
2377 data->start = (new_prot ? end : -1ul);
2378 data->prot = new_prot;
2379
2380 return 0;
2381}
2382
2383static int walk_memory_regions_1(struct walk_memory_regions_data *data,
Paul Brookb480d9b2010-03-12 23:23:29 +00002384 abi_ulong base, int level, void **lp)
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002385{
Paul Brookb480d9b2010-03-12 23:23:29 +00002386 abi_ulong pa;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002387 int i, rc;
2388
2389 if (*lp == NULL) {
2390 return walk_memory_regions_end(data, base, 0);
2391 }
2392
2393 if (level == 0) {
2394 PageDesc *pd = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +00002395 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002396 int prot = pd[i].flags;
2397
2398 pa = base | (i << TARGET_PAGE_BITS);
2399 if (prot != data->prot) {
2400 rc = walk_memory_regions_end(data, pa, prot);
2401 if (rc != 0) {
2402 return rc;
2403 }
2404 }
2405 }
2406 } else {
2407 void **pp = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +00002408 for (i = 0; i < L2_SIZE; ++i) {
Paul Brookb480d9b2010-03-12 23:23:29 +00002409 pa = base | ((abi_ulong)i <<
2410 (TARGET_PAGE_BITS + L2_BITS * level));
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002411 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2412 if (rc != 0) {
2413 return rc;
2414 }
2415 }
2416 }
2417
2418 return 0;
2419}
2420
2421int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2422{
2423 struct walk_memory_regions_data data;
2424 unsigned long i;
2425
2426 data.fn = fn;
2427 data.priv = priv;
2428 data.start = -1ul;
2429 data.prot = 0;
2430
2431 for (i = 0; i < V_L1_SIZE; i++) {
Paul Brookb480d9b2010-03-12 23:23:29 +00002432 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002433 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2434 if (rc != 0) {
2435 return rc;
2436 }
2437 }
2438
2439 return walk_memory_regions_end(&data, 0, 0);
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002440}
2441
Paul Brookb480d9b2010-03-12 23:23:29 +00002442static int dump_region(void *priv, abi_ulong start,
2443 abi_ulong end, unsigned long prot)
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002444{
2445 FILE *f = (FILE *)priv;
2446
Paul Brookb480d9b2010-03-12 23:23:29 +00002447 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2448 " "TARGET_ABI_FMT_lx" %c%c%c\n",
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002449 start, end, end - start,
2450 ((prot & PAGE_READ) ? 'r' : '-'),
2451 ((prot & PAGE_WRITE) ? 'w' : '-'),
2452 ((prot & PAGE_EXEC) ? 'x' : '-'));
2453
2454 return (0);
2455}
2456
2457/* dump memory mappings */
2458void page_dump(FILE *f)
2459{
2460 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2461 "start", "end", "size", "prot");
2462 walk_memory_regions(f, dump_region);
bellard33417e72003-08-10 21:47:01 +00002463}
2464
pbrook53a59602006-03-25 19:31:22 +00002465int page_get_flags(target_ulong address)
bellard33417e72003-08-10 21:47:01 +00002466{
bellard9fa3e852004-01-04 18:06:42 +00002467 PageDesc *p;
2468
2469 p = page_find(address >> TARGET_PAGE_BITS);
bellard33417e72003-08-10 21:47:01 +00002470 if (!p)
bellard9fa3e852004-01-04 18:06:42 +00002471 return 0;
2472 return p->flags;
bellard33417e72003-08-10 21:47:01 +00002473}
2474
Richard Henderson376a7902010-03-10 15:57:04 -08002475/* Modify the flags of a page and invalidate the code if necessary.
2476 The flag PAGE_WRITE_ORG is positioned automatically depending
2477 on PAGE_WRITE. The mmap_lock should already be held. */
pbrook53a59602006-03-25 19:31:22 +00002478void page_set_flags(target_ulong start, target_ulong end, int flags)
bellard9fa3e852004-01-04 18:06:42 +00002479{
Richard Henderson376a7902010-03-10 15:57:04 -08002480 target_ulong addr, len;
bellard9fa3e852004-01-04 18:06:42 +00002481
Richard Henderson376a7902010-03-10 15:57:04 -08002482 /* This function should never be called with addresses outside the
2483 guest address space. If this assert fires, it probably indicates
2484 a missing call to h2g_valid. */
Paul Brookb480d9b2010-03-12 23:23:29 +00002485#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2486 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Richard Henderson376a7902010-03-10 15:57:04 -08002487#endif
2488 assert(start < end);
2489
bellard9fa3e852004-01-04 18:06:42 +00002490 start = start & TARGET_PAGE_MASK;
2491 end = TARGET_PAGE_ALIGN(end);
Richard Henderson376a7902010-03-10 15:57:04 -08002492
2493 if (flags & PAGE_WRITE) {
bellard9fa3e852004-01-04 18:06:42 +00002494 flags |= PAGE_WRITE_ORG;
Richard Henderson376a7902010-03-10 15:57:04 -08002495 }
2496
2497 for (addr = start, len = end - start;
2498 len != 0;
2499 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2500 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2501
2502 /* If the write protection bit is set, then we invalidate
2503 the code inside. */
ths5fafdf22007-09-16 21:08:06 +00002504 if (!(p->flags & PAGE_WRITE) &&
bellard9fa3e852004-01-04 18:06:42 +00002505 (flags & PAGE_WRITE) &&
2506 p->first_tb) {
bellardd720b932004-04-25 17:57:43 +00002507 tb_invalidate_phys_page(addr, 0, NULL);
bellard9fa3e852004-01-04 18:06:42 +00002508 }
2509 p->flags = flags;
2510 }
bellard9fa3e852004-01-04 18:06:42 +00002511}
2512
ths3d97b402007-11-02 19:02:07 +00002513int page_check_range(target_ulong start, target_ulong len, int flags)
2514{
2515 PageDesc *p;
2516 target_ulong end;
2517 target_ulong addr;
2518
Richard Henderson376a7902010-03-10 15:57:04 -08002519 /* This function should never be called with addresses outside the
2520 guest address space. If this assert fires, it probably indicates
2521 a missing call to h2g_valid. */
Blue Swirl338e9e62010-03-13 09:48:08 +00002522#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2523 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Richard Henderson376a7902010-03-10 15:57:04 -08002524#endif
2525
Richard Henderson3e0650a2010-03-29 10:54:42 -07002526 if (len == 0) {
2527 return 0;
2528 }
Richard Henderson376a7902010-03-10 15:57:04 -08002529 if (start + len - 1 < start) {
2530 /* We've wrapped around. */
balrog55f280c2008-10-28 10:24:11 +00002531 return -1;
Richard Henderson376a7902010-03-10 15:57:04 -08002532 }
balrog55f280c2008-10-28 10:24:11 +00002533
ths3d97b402007-11-02 19:02:07 +00002534 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2535 start = start & TARGET_PAGE_MASK;
2536
Richard Henderson376a7902010-03-10 15:57:04 -08002537 for (addr = start, len = end - start;
2538 len != 0;
2539 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
ths3d97b402007-11-02 19:02:07 +00002540 p = page_find(addr >> TARGET_PAGE_BITS);
2541 if( !p )
2542 return -1;
2543 if( !(p->flags & PAGE_VALID) )
2544 return -1;
2545
bellarddae32702007-11-14 10:51:00 +00002546 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
ths3d97b402007-11-02 19:02:07 +00002547 return -1;
bellarddae32702007-11-14 10:51:00 +00002548 if (flags & PAGE_WRITE) {
2549 if (!(p->flags & PAGE_WRITE_ORG))
2550 return -1;
2551 /* unprotect the page if it was put read-only because it
2552 contains translated code */
2553 if (!(p->flags & PAGE_WRITE)) {
2554 if (!page_unprotect(addr, 0, NULL))
2555 return -1;
2556 }
2557 return 0;
2558 }
ths3d97b402007-11-02 19:02:07 +00002559 }
2560 return 0;
2561}
2562
bellard9fa3e852004-01-04 18:06:42 +00002563/* called from signal handler: invalidate the code and unprotect the
Stuart Bradyccbb4d42009-05-03 12:15:06 +01002564 page. Return TRUE if the fault was successfully handled. */
pbrook53a59602006-03-25 19:31:22 +00002565int page_unprotect(target_ulong address, unsigned long pc, void *puc)
bellard9fa3e852004-01-04 18:06:42 +00002566{
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002567 unsigned int prot;
2568 PageDesc *p;
pbrook53a59602006-03-25 19:31:22 +00002569 target_ulong host_start, host_end, addr;
bellard9fa3e852004-01-04 18:06:42 +00002570
pbrookc8a706f2008-06-02 16:16:42 +00002571 /* Technically this isn't safe inside a signal handler. However we
2572 know this only ever happens in a synchronous SEGV handler, so in
2573 practice it seems to be ok. */
2574 mmap_lock();
2575
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002576 p = page_find(address >> TARGET_PAGE_BITS);
2577 if (!p) {
pbrookc8a706f2008-06-02 16:16:42 +00002578 mmap_unlock();
bellard9fa3e852004-01-04 18:06:42 +00002579 return 0;
pbrookc8a706f2008-06-02 16:16:42 +00002580 }
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002581
bellard9fa3e852004-01-04 18:06:42 +00002582 /* if the page was really writable, then we change its
2583 protection back to writable */
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002584 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2585 host_start = address & qemu_host_page_mask;
2586 host_end = host_start + qemu_host_page_size;
2587
2588 prot = 0;
2589 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2590 p = page_find(addr >> TARGET_PAGE_BITS);
2591 p->flags |= PAGE_WRITE;
2592 prot |= p->flags;
2593
bellard9fa3e852004-01-04 18:06:42 +00002594 /* and since the content will be modified, we must invalidate
2595 the corresponding translated code. */
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002596 tb_invalidate_phys_page(addr, pc, puc);
bellard9fa3e852004-01-04 18:06:42 +00002597#ifdef DEBUG_TB_CHECK
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002598 tb_invalidate_check(addr);
bellard9fa3e852004-01-04 18:06:42 +00002599#endif
bellard9fa3e852004-01-04 18:06:42 +00002600 }
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002601 mprotect((void *)g2h(host_start), qemu_host_page_size,
2602 prot & PAGE_BITS);
2603
2604 mmap_unlock();
2605 return 1;
bellard9fa3e852004-01-04 18:06:42 +00002606 }
pbrookc8a706f2008-06-02 16:16:42 +00002607 mmap_unlock();
bellard9fa3e852004-01-04 18:06:42 +00002608 return 0;
2609}
2610
bellard6a00d602005-11-21 23:25:50 +00002611static inline void tlb_set_dirty(CPUState *env,
2612 unsigned long addr, target_ulong vaddr)
bellard1ccde1c2004-02-06 19:46:14 +00002613{
2614}
bellard9fa3e852004-01-04 18:06:42 +00002615#endif /* defined(CONFIG_USER_ONLY) */
2616
pbrooke2eef172008-06-08 01:09:01 +00002617#if !defined(CONFIG_USER_ONLY)
pbrook8da3ff12008-12-01 18:59:50 +00002618
Paul Brookc04b2b72010-03-01 03:31:14 +00002619#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2620typedef struct subpage_t {
2621 target_phys_addr_t base;
Richard Hendersonf6405242010-04-22 16:47:31 -07002622 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2623 ram_addr_t region_offset[TARGET_PAGE_SIZE];
Paul Brookc04b2b72010-03-01 03:31:14 +00002624} subpage_t;
2625
Anthony Liguoric227f092009-10-01 16:12:16 -05002626static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2627 ram_addr_t memory, ram_addr_t region_offset);
Richard Hendersonf6405242010-04-22 16:47:31 -07002628static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2629 ram_addr_t orig_memory,
2630 ram_addr_t region_offset);
blueswir1db7b5422007-05-26 17:36:03 +00002631#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2632 need_subpage) \
2633 do { \
2634 if (addr > start_addr) \
2635 start_addr2 = 0; \
2636 else { \
2637 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2638 if (start_addr2 > 0) \
2639 need_subpage = 1; \
2640 } \
2641 \
blueswir149e9fba2007-05-30 17:25:06 +00002642 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
blueswir1db7b5422007-05-26 17:36:03 +00002643 end_addr2 = TARGET_PAGE_SIZE - 1; \
2644 else { \
2645 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2646 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2647 need_subpage = 1; \
2648 } \
2649 } while (0)
2650
Michael S. Tsirkin8f2498f2009-09-29 18:53:16 +02002651/* register physical memory.
2652 For RAM, 'size' must be a multiple of the target page size.
2653 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
pbrook8da3ff12008-12-01 18:59:50 +00002654 io memory page. The address used when calling the IO function is
2655 the offset from the start of the region, plus region_offset. Both
Stuart Bradyccbb4d42009-05-03 12:15:06 +01002656 start_addr and region_offset are rounded down to a page boundary
pbrook8da3ff12008-12-01 18:59:50 +00002657 before calculating this offset. This should not be a problem unless
2658 the low bits of start_addr and region_offset differ. */
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03002659void cpu_register_physical_memory_log(target_phys_addr_t start_addr,
Anthony Liguoric227f092009-10-01 16:12:16 -05002660 ram_addr_t size,
2661 ram_addr_t phys_offset,
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03002662 ram_addr_t region_offset,
2663 bool log_dirty)
bellard33417e72003-08-10 21:47:01 +00002664{
Anthony Liguoric227f092009-10-01 16:12:16 -05002665 target_phys_addr_t addr, end_addr;
bellard92e873b2004-05-21 14:52:29 +00002666 PhysPageDesc *p;
bellard9d420372006-06-25 22:25:22 +00002667 CPUState *env;
Anthony Liguoric227f092009-10-01 16:12:16 -05002668 ram_addr_t orig_size = size;
Richard Hendersonf6405242010-04-22 16:47:31 -07002669 subpage_t *subpage;
bellard33417e72003-08-10 21:47:01 +00002670
Edgar E. Iglesias3b8e6a22011-04-05 13:00:36 +02002671 assert(size);
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03002672 cpu_notify_set_memory(start_addr, size, phys_offset, log_dirty);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002673
pbrook67c4d232009-02-23 13:16:07 +00002674 if (phys_offset == IO_MEM_UNASSIGNED) {
2675 region_offset = start_addr;
2676 }
pbrook8da3ff12008-12-01 18:59:50 +00002677 region_offset &= TARGET_PAGE_MASK;
bellard5fd386f2004-05-23 21:11:22 +00002678 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
Anthony Liguoric227f092009-10-01 16:12:16 -05002679 end_addr = start_addr + (target_phys_addr_t)size;
Edgar E. Iglesias3b8e6a22011-04-05 13:00:36 +02002680
2681 addr = start_addr;
2682 do {
blueswir1db7b5422007-05-26 17:36:03 +00002683 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2684 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
Anthony Liguoric227f092009-10-01 16:12:16 -05002685 ram_addr_t orig_memory = p->phys_offset;
2686 target_phys_addr_t start_addr2, end_addr2;
blueswir1db7b5422007-05-26 17:36:03 +00002687 int need_subpage = 0;
2688
2689 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2690 need_subpage);
Richard Hendersonf6405242010-04-22 16:47:31 -07002691 if (need_subpage) {
blueswir1db7b5422007-05-26 17:36:03 +00002692 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2693 subpage = subpage_init((addr & TARGET_PAGE_MASK),
pbrook8da3ff12008-12-01 18:59:50 +00002694 &p->phys_offset, orig_memory,
2695 p->region_offset);
blueswir1db7b5422007-05-26 17:36:03 +00002696 } else {
2697 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2698 >> IO_MEM_SHIFT];
2699 }
pbrook8da3ff12008-12-01 18:59:50 +00002700 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2701 region_offset);
2702 p->region_offset = 0;
blueswir1db7b5422007-05-26 17:36:03 +00002703 } else {
2704 p->phys_offset = phys_offset;
2705 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2706 (phys_offset & IO_MEM_ROMD))
2707 phys_offset += TARGET_PAGE_SIZE;
2708 }
2709 } else {
2710 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2711 p->phys_offset = phys_offset;
pbrook8da3ff12008-12-01 18:59:50 +00002712 p->region_offset = region_offset;
blueswir1db7b5422007-05-26 17:36:03 +00002713 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
pbrook8da3ff12008-12-01 18:59:50 +00002714 (phys_offset & IO_MEM_ROMD)) {
blueswir1db7b5422007-05-26 17:36:03 +00002715 phys_offset += TARGET_PAGE_SIZE;
pbrook0e8f0962008-12-02 09:02:15 +00002716 } else {
Anthony Liguoric227f092009-10-01 16:12:16 -05002717 target_phys_addr_t start_addr2, end_addr2;
blueswir1db7b5422007-05-26 17:36:03 +00002718 int need_subpage = 0;
2719
2720 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2721 end_addr2, need_subpage);
2722
Richard Hendersonf6405242010-04-22 16:47:31 -07002723 if (need_subpage) {
blueswir1db7b5422007-05-26 17:36:03 +00002724 subpage = subpage_init((addr & TARGET_PAGE_MASK),
pbrook8da3ff12008-12-01 18:59:50 +00002725 &p->phys_offset, IO_MEM_UNASSIGNED,
pbrook67c4d232009-02-23 13:16:07 +00002726 addr & TARGET_PAGE_MASK);
blueswir1db7b5422007-05-26 17:36:03 +00002727 subpage_register(subpage, start_addr2, end_addr2,
pbrook8da3ff12008-12-01 18:59:50 +00002728 phys_offset, region_offset);
2729 p->region_offset = 0;
blueswir1db7b5422007-05-26 17:36:03 +00002730 }
2731 }
2732 }
pbrook8da3ff12008-12-01 18:59:50 +00002733 region_offset += TARGET_PAGE_SIZE;
Edgar E. Iglesias3b8e6a22011-04-05 13:00:36 +02002734 addr += TARGET_PAGE_SIZE;
2735 } while (addr != end_addr);
ths3b46e622007-09-17 08:09:54 +00002736
bellard9d420372006-06-25 22:25:22 +00002737 /* since each CPU stores ram addresses in its TLB cache, we must
2738 reset the modified entries */
2739 /* XXX: slow ! */
2740 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2741 tlb_flush(env, 1);
2742 }
bellard33417e72003-08-10 21:47:01 +00002743}
2744
bellardba863452006-09-24 18:41:10 +00002745/* XXX: temporary until new memory mapping API */
Anthony Liguoric227f092009-10-01 16:12:16 -05002746ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
bellardba863452006-09-24 18:41:10 +00002747{
2748 PhysPageDesc *p;
2749
2750 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2751 if (!p)
2752 return IO_MEM_UNASSIGNED;
2753 return p->phys_offset;
2754}
2755
Anthony Liguoric227f092009-10-01 16:12:16 -05002756void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
aliguorif65ed4c2008-12-09 20:09:57 +00002757{
2758 if (kvm_enabled())
2759 kvm_coalesce_mmio_region(addr, size);
2760}
2761
Anthony Liguoric227f092009-10-01 16:12:16 -05002762void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
aliguorif65ed4c2008-12-09 20:09:57 +00002763{
2764 if (kvm_enabled())
2765 kvm_uncoalesce_mmio_region(addr, size);
2766}
2767
Sheng Yang62a27442010-01-26 19:21:16 +08002768void qemu_flush_coalesced_mmio_buffer(void)
2769{
2770 if (kvm_enabled())
2771 kvm_flush_coalesced_mmio_buffer();
2772}
2773
Marcelo Tosattic9027602010-03-01 20:25:08 -03002774#if defined(__linux__) && !defined(TARGET_S390X)
2775
2776#include <sys/vfs.h>
2777
2778#define HUGETLBFS_MAGIC 0x958458f6
2779
2780static long gethugepagesize(const char *path)
2781{
2782 struct statfs fs;
2783 int ret;
2784
2785 do {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002786 ret = statfs(path, &fs);
Marcelo Tosattic9027602010-03-01 20:25:08 -03002787 } while (ret != 0 && errno == EINTR);
2788
2789 if (ret != 0) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002790 perror(path);
2791 return 0;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002792 }
2793
2794 if (fs.f_type != HUGETLBFS_MAGIC)
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002795 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
Marcelo Tosattic9027602010-03-01 20:25:08 -03002796
2797 return fs.f_bsize;
2798}
2799
Alex Williamson04b16652010-07-02 11:13:17 -06002800static void *file_ram_alloc(RAMBlock *block,
2801 ram_addr_t memory,
2802 const char *path)
Marcelo Tosattic9027602010-03-01 20:25:08 -03002803{
2804 char *filename;
2805 void *area;
2806 int fd;
2807#ifdef MAP_POPULATE
2808 int flags;
2809#endif
2810 unsigned long hpagesize;
2811
2812 hpagesize = gethugepagesize(path);
2813 if (!hpagesize) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002814 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002815 }
2816
2817 if (memory < hpagesize) {
2818 return NULL;
2819 }
2820
2821 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2822 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2823 return NULL;
2824 }
2825
2826 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002827 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002828 }
2829
2830 fd = mkstemp(filename);
2831 if (fd < 0) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002832 perror("unable to create backing store for hugepages");
2833 free(filename);
2834 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002835 }
2836 unlink(filename);
2837 free(filename);
2838
2839 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2840
2841 /*
2842 * ftruncate is not supported by hugetlbfs in older
2843 * hosts, so don't bother bailing out on errors.
2844 * If anything goes wrong with it under other filesystems,
2845 * mmap will fail.
2846 */
2847 if (ftruncate(fd, memory))
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002848 perror("ftruncate");
Marcelo Tosattic9027602010-03-01 20:25:08 -03002849
2850#ifdef MAP_POPULATE
2851 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2852 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2853 * to sidestep this quirk.
2854 */
2855 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2856 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2857#else
2858 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2859#endif
2860 if (area == MAP_FAILED) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002861 perror("file_ram_alloc: can't mmap RAM pages");
2862 close(fd);
2863 return (NULL);
Marcelo Tosattic9027602010-03-01 20:25:08 -03002864 }
Alex Williamson04b16652010-07-02 11:13:17 -06002865 block->fd = fd;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002866 return area;
2867}
2868#endif
2869
Alex Williamsond17b5282010-06-25 11:08:38 -06002870static ram_addr_t find_ram_offset(ram_addr_t size)
2871{
Alex Williamson04b16652010-07-02 11:13:17 -06002872 RAMBlock *block, *next_block;
Anthony PERARDf15fbc42011-07-20 08:17:42 +00002873 ram_addr_t offset = 0, mingap = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06002874
2875 if (QLIST_EMPTY(&ram_list.blocks))
2876 return 0;
2877
2878 QLIST_FOREACH(block, &ram_list.blocks, next) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00002879 ram_addr_t end, next = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06002880
2881 end = block->offset + block->length;
2882
2883 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2884 if (next_block->offset >= end) {
2885 next = MIN(next, next_block->offset);
2886 }
2887 }
2888 if (next - end >= size && next - end < mingap) {
2889 offset = end;
2890 mingap = next - end;
2891 }
2892 }
2893 return offset;
2894}
2895
2896static ram_addr_t last_ram_offset(void)
2897{
Alex Williamsond17b5282010-06-25 11:08:38 -06002898 RAMBlock *block;
2899 ram_addr_t last = 0;
2900
2901 QLIST_FOREACH(block, &ram_list.blocks, next)
2902 last = MAX(last, block->offset + block->length);
2903
2904 return last;
2905}
2906
Cam Macdonell84b89d72010-07-26 18:10:57 -06002907ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002908 ram_addr_t size, void *host)
Cam Macdonell84b89d72010-07-26 18:10:57 -06002909{
2910 RAMBlock *new_block, *block;
2911
2912 size = TARGET_PAGE_ALIGN(size);
2913 new_block = qemu_mallocz(sizeof(*new_block));
2914
2915 if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2916 char *id = dev->parent_bus->info->get_dev_path(dev);
2917 if (id) {
2918 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2919 qemu_free(id);
2920 }
2921 }
2922 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2923
2924 QLIST_FOREACH(block, &ram_list.blocks, next) {
2925 if (!strcmp(block->idstr, new_block->idstr)) {
2926 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2927 new_block->idstr);
2928 abort();
2929 }
2930 }
2931
Jun Nakajima432d2682010-08-31 16:41:25 +01002932 new_block->offset = find_ram_offset(size);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002933 if (host) {
2934 new_block->host = host;
Huang Yingcd19cfa2011-03-02 08:56:19 +01002935 new_block->flags |= RAM_PREALLOC_MASK;
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002936 } else {
2937 if (mem_path) {
2938#if defined (__linux__) && !defined(TARGET_S390X)
2939 new_block->host = file_ram_alloc(new_block, size, mem_path);
2940 if (!new_block->host) {
2941 new_block->host = qemu_vmalloc(size);
Andreas Färbere78815a2010-09-25 11:26:05 +00002942 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002943 }
2944#else
2945 fprintf(stderr, "-mem-path option unsupported\n");
2946 exit(1);
2947#endif
2948 } else {
2949#if defined(TARGET_S390X) && defined(CONFIG_KVM)
Christian Borntraegerff836782011-05-10 14:49:10 +02002950 /* S390 KVM requires the topmost vma of the RAM to be smaller than
2951 an system defined value, which is at least 256GB. Larger systems
2952 have larger values. We put the guest between the end of data
2953 segment (system break) and this value. We use 32GB as a base to
2954 have enough room for the system break to grow. */
2955 new_block->host = mmap((void*)0x800000000, size,
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002956 PROT_EXEC|PROT_READ|PROT_WRITE,
Christian Borntraegerff836782011-05-10 14:49:10 +02002957 MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
Alexander Graffb8b2732011-05-20 17:33:28 +02002958 if (new_block->host == MAP_FAILED) {
2959 fprintf(stderr, "Allocating RAM failed\n");
2960 abort();
2961 }
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002962#else
Jan Kiszka868bb332011-06-21 22:59:09 +02002963 if (xen_enabled()) {
Jun Nakajima432d2682010-08-31 16:41:25 +01002964 xen_ram_alloc(new_block->offset, size);
2965 } else {
2966 new_block->host = qemu_vmalloc(size);
2967 }
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002968#endif
Andreas Färbere78815a2010-09-25 11:26:05 +00002969 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002970 }
2971 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06002972 new_block->length = size;
2973
2974 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2975
2976 ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2977 last_ram_offset() >> TARGET_PAGE_BITS);
2978 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2979 0xff, size >> TARGET_PAGE_BITS);
2980
2981 if (kvm_enabled())
2982 kvm_setup_guest_memory(new_block->host, size);
2983
2984 return new_block->offset;
2985}
2986
Alex Williamson1724f042010-06-25 11:09:35 -06002987ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
pbrook94a6b542009-04-11 17:15:54 +00002988{
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002989 return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
pbrook94a6b542009-04-11 17:15:54 +00002990}
bellarde9a1ab12007-02-08 23:08:38 +00002991
Alex Williamson1f2e98b2011-05-03 12:48:09 -06002992void qemu_ram_free_from_ptr(ram_addr_t addr)
2993{
2994 RAMBlock *block;
2995
2996 QLIST_FOREACH(block, &ram_list.blocks, next) {
2997 if (addr == block->offset) {
2998 QLIST_REMOVE(block, next);
2999 qemu_free(block);
3000 return;
3001 }
3002 }
3003}
3004
Anthony Liguoric227f092009-10-01 16:12:16 -05003005void qemu_ram_free(ram_addr_t addr)
bellarde9a1ab12007-02-08 23:08:38 +00003006{
Alex Williamson04b16652010-07-02 11:13:17 -06003007 RAMBlock *block;
3008
3009 QLIST_FOREACH(block, &ram_list.blocks, next) {
3010 if (addr == block->offset) {
3011 QLIST_REMOVE(block, next);
Huang Yingcd19cfa2011-03-02 08:56:19 +01003012 if (block->flags & RAM_PREALLOC_MASK) {
3013 ;
3014 } else if (mem_path) {
Alex Williamson04b16652010-07-02 11:13:17 -06003015#if defined (__linux__) && !defined(TARGET_S390X)
3016 if (block->fd) {
3017 munmap(block->host, block->length);
3018 close(block->fd);
3019 } else {
3020 qemu_vfree(block->host);
3021 }
Jan Kiszkafd28aa12011-03-15 12:26:14 +01003022#else
3023 abort();
Alex Williamson04b16652010-07-02 11:13:17 -06003024#endif
3025 } else {
3026#if defined(TARGET_S390X) && defined(CONFIG_KVM)
3027 munmap(block->host, block->length);
3028#else
Jan Kiszka868bb332011-06-21 22:59:09 +02003029 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003030 xen_invalidate_map_cache_entry(block->host);
Jun Nakajima432d2682010-08-31 16:41:25 +01003031 } else {
3032 qemu_vfree(block->host);
3033 }
Alex Williamson04b16652010-07-02 11:13:17 -06003034#endif
3035 }
3036 qemu_free(block);
3037 return;
3038 }
3039 }
3040
bellarde9a1ab12007-02-08 23:08:38 +00003041}
3042
Huang Yingcd19cfa2011-03-02 08:56:19 +01003043#ifndef _WIN32
3044void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
3045{
3046 RAMBlock *block;
3047 ram_addr_t offset;
3048 int flags;
3049 void *area, *vaddr;
3050
3051 QLIST_FOREACH(block, &ram_list.blocks, next) {
3052 offset = addr - block->offset;
3053 if (offset < block->length) {
3054 vaddr = block->host + offset;
3055 if (block->flags & RAM_PREALLOC_MASK) {
3056 ;
3057 } else {
3058 flags = MAP_FIXED;
3059 munmap(vaddr, length);
3060 if (mem_path) {
3061#if defined(__linux__) && !defined(TARGET_S390X)
3062 if (block->fd) {
3063#ifdef MAP_POPULATE
3064 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
3065 MAP_PRIVATE;
3066#else
3067 flags |= MAP_PRIVATE;
3068#endif
3069 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3070 flags, block->fd, offset);
3071 } else {
3072 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3073 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3074 flags, -1, 0);
3075 }
Jan Kiszkafd28aa12011-03-15 12:26:14 +01003076#else
3077 abort();
Huang Yingcd19cfa2011-03-02 08:56:19 +01003078#endif
3079 } else {
3080#if defined(TARGET_S390X) && defined(CONFIG_KVM)
3081 flags |= MAP_SHARED | MAP_ANONYMOUS;
3082 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
3083 flags, -1, 0);
3084#else
3085 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3086 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3087 flags, -1, 0);
3088#endif
3089 }
3090 if (area != vaddr) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00003091 fprintf(stderr, "Could not remap addr: "
3092 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
Huang Yingcd19cfa2011-03-02 08:56:19 +01003093 length, addr);
3094 exit(1);
3095 }
3096 qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
3097 }
3098 return;
3099 }
3100 }
3101}
3102#endif /* !_WIN32 */
3103
pbrookdc828ca2009-04-09 22:21:07 +00003104/* Return a host pointer to ram allocated with qemu_ram_alloc.
pbrook5579c7f2009-04-11 14:47:08 +00003105 With the exception of the softmmu code in this file, this should
3106 only be used for local memory (e.g. video ram) that the device owns,
3107 and knows it isn't going to access beyond the end of the block.
3108
3109 It should not be used for general purpose DMA.
3110 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3111 */
Anthony Liguoric227f092009-10-01 16:12:16 -05003112void *qemu_get_ram_ptr(ram_addr_t addr)
pbrookdc828ca2009-04-09 22:21:07 +00003113{
pbrook94a6b542009-04-11 17:15:54 +00003114 RAMBlock *block;
3115
Alex Williamsonf471a172010-06-11 11:11:42 -06003116 QLIST_FOREACH(block, &ram_list.blocks, next) {
3117 if (addr - block->offset < block->length) {
Vincent Palatin7d82af32011-03-10 15:47:46 -05003118 /* Move this entry to to start of the list. */
3119 if (block != QLIST_FIRST(&ram_list.blocks)) {
3120 QLIST_REMOVE(block, next);
3121 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
3122 }
Jan Kiszka868bb332011-06-21 22:59:09 +02003123 if (xen_enabled()) {
Jun Nakajima432d2682010-08-31 16:41:25 +01003124 /* We need to check if the requested address is in the RAM
3125 * because we don't want to map the entire memory in QEMU.
Stefano Stabellini712c2b42011-05-19 18:35:46 +01003126 * In that case just map until the end of the page.
Jun Nakajima432d2682010-08-31 16:41:25 +01003127 */
3128 if (block->offset == 0) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003129 return xen_map_cache(addr, 0, 0);
Jun Nakajima432d2682010-08-31 16:41:25 +01003130 } else if (block->host == NULL) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003131 block->host =
3132 xen_map_cache(block->offset, block->length, 1);
Jun Nakajima432d2682010-08-31 16:41:25 +01003133 }
3134 }
Alex Williamsonf471a172010-06-11 11:11:42 -06003135 return block->host + (addr - block->offset);
3136 }
pbrook94a6b542009-04-11 17:15:54 +00003137 }
Alex Williamsonf471a172010-06-11 11:11:42 -06003138
3139 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3140 abort();
3141
3142 return NULL;
pbrookdc828ca2009-04-09 22:21:07 +00003143}
3144
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02003145/* Return a host pointer to ram allocated with qemu_ram_alloc.
3146 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3147 */
3148void *qemu_safe_ram_ptr(ram_addr_t addr)
3149{
3150 RAMBlock *block;
3151
3152 QLIST_FOREACH(block, &ram_list.blocks, next) {
3153 if (addr - block->offset < block->length) {
Jan Kiszka868bb332011-06-21 22:59:09 +02003154 if (xen_enabled()) {
Jun Nakajima432d2682010-08-31 16:41:25 +01003155 /* We need to check if the requested address is in the RAM
3156 * because we don't want to map the entire memory in QEMU.
Stefano Stabellini712c2b42011-05-19 18:35:46 +01003157 * In that case just map until the end of the page.
Jun Nakajima432d2682010-08-31 16:41:25 +01003158 */
3159 if (block->offset == 0) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003160 return xen_map_cache(addr, 0, 0);
Jun Nakajima432d2682010-08-31 16:41:25 +01003161 } else if (block->host == NULL) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003162 block->host =
3163 xen_map_cache(block->offset, block->length, 1);
Jun Nakajima432d2682010-08-31 16:41:25 +01003164 }
3165 }
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02003166 return block->host + (addr - block->offset);
3167 }
3168 }
3169
3170 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3171 abort();
3172
3173 return NULL;
3174}
3175
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01003176/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
3177 * but takes a size argument */
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01003178void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01003179{
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01003180 if (*size == 0) {
3181 return NULL;
3182 }
Jan Kiszka868bb332011-06-21 22:59:09 +02003183 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003184 return xen_map_cache(addr, *size, 1);
Jan Kiszka868bb332011-06-21 22:59:09 +02003185 } else {
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01003186 RAMBlock *block;
3187
3188 QLIST_FOREACH(block, &ram_list.blocks, next) {
3189 if (addr - block->offset < block->length) {
3190 if (addr - block->offset + *size > block->length)
3191 *size = block->length - addr + block->offset;
3192 return block->host + (addr - block->offset);
3193 }
3194 }
3195
3196 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3197 abort();
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01003198 }
3199}
3200
Anthony PERARD050a0dd2010-09-16 13:57:49 +01003201void qemu_put_ram_ptr(void *addr)
3202{
3203 trace_qemu_put_ram_ptr(addr);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01003204}
3205
Marcelo Tosattie8902612010-10-11 15:31:19 -03003206int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
pbrook5579c7f2009-04-11 14:47:08 +00003207{
pbrook94a6b542009-04-11 17:15:54 +00003208 RAMBlock *block;
3209 uint8_t *host = ptr;
3210
Jan Kiszka868bb332011-06-21 22:59:09 +02003211 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003212 *ram_addr = xen_ram_addr_from_mapcache(ptr);
Stefano Stabellini712c2b42011-05-19 18:35:46 +01003213 return 0;
3214 }
3215
Alex Williamsonf471a172010-06-11 11:11:42 -06003216 QLIST_FOREACH(block, &ram_list.blocks, next) {
Jun Nakajima432d2682010-08-31 16:41:25 +01003217 /* This case append when the block is not mapped. */
3218 if (block->host == NULL) {
3219 continue;
3220 }
Alex Williamsonf471a172010-06-11 11:11:42 -06003221 if (host - block->host < block->length) {
Marcelo Tosattie8902612010-10-11 15:31:19 -03003222 *ram_addr = block->offset + (host - block->host);
3223 return 0;
Alex Williamsonf471a172010-06-11 11:11:42 -06003224 }
pbrook94a6b542009-04-11 17:15:54 +00003225 }
Jun Nakajima432d2682010-08-31 16:41:25 +01003226
Marcelo Tosattie8902612010-10-11 15:31:19 -03003227 return -1;
3228}
Alex Williamsonf471a172010-06-11 11:11:42 -06003229
Marcelo Tosattie8902612010-10-11 15:31:19 -03003230/* Some of the softmmu routines need to translate from a host pointer
3231 (typically a TLB entry) back to a ram offset. */
3232ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
3233{
3234 ram_addr_t ram_addr;
Alex Williamsonf471a172010-06-11 11:11:42 -06003235
Marcelo Tosattie8902612010-10-11 15:31:19 -03003236 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
3237 fprintf(stderr, "Bad ram pointer %p\n", ptr);
3238 abort();
3239 }
3240 return ram_addr;
pbrook5579c7f2009-04-11 14:47:08 +00003241}
3242
Anthony Liguoric227f092009-10-01 16:12:16 -05003243static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
bellard33417e72003-08-10 21:47:01 +00003244{
pbrook67d3b952006-12-18 05:03:52 +00003245#ifdef DEBUG_UNASSIGNED
blueswir1ab3d1722007-11-04 07:31:40 +00003246 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
pbrook67d3b952006-12-18 05:03:52 +00003247#endif
Richard Henderson5b450402011-04-18 16:13:12 -07003248#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
Blue Swirlb14ef7c2011-07-03 08:53:46 +00003249 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 1);
blueswir1e18231a2008-10-06 18:46:28 +00003250#endif
3251 return 0;
3252}
3253
Anthony Liguoric227f092009-10-01 16:12:16 -05003254static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
blueswir1e18231a2008-10-06 18:46:28 +00003255{
3256#ifdef DEBUG_UNASSIGNED
3257 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3258#endif
Richard Henderson5b450402011-04-18 16:13:12 -07003259#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
Blue Swirlb14ef7c2011-07-03 08:53:46 +00003260 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 2);
blueswir1e18231a2008-10-06 18:46:28 +00003261#endif
3262 return 0;
3263}
3264
Anthony Liguoric227f092009-10-01 16:12:16 -05003265static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
blueswir1e18231a2008-10-06 18:46:28 +00003266{
3267#ifdef DEBUG_UNASSIGNED
3268 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3269#endif
Richard Henderson5b450402011-04-18 16:13:12 -07003270#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
Blue Swirlb14ef7c2011-07-03 08:53:46 +00003271 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 4);
blueswir1b4f0a312007-05-06 17:59:24 +00003272#endif
bellard33417e72003-08-10 21:47:01 +00003273 return 0;
3274}
3275
Anthony Liguoric227f092009-10-01 16:12:16 -05003276static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
bellard33417e72003-08-10 21:47:01 +00003277{
pbrook67d3b952006-12-18 05:03:52 +00003278#ifdef DEBUG_UNASSIGNED
blueswir1ab3d1722007-11-04 07:31:40 +00003279 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
pbrook67d3b952006-12-18 05:03:52 +00003280#endif
Richard Henderson5b450402011-04-18 16:13:12 -07003281#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
Blue Swirlb14ef7c2011-07-03 08:53:46 +00003282 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 1);
blueswir1e18231a2008-10-06 18:46:28 +00003283#endif
3284}
3285
Anthony Liguoric227f092009-10-01 16:12:16 -05003286static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
blueswir1e18231a2008-10-06 18:46:28 +00003287{
3288#ifdef DEBUG_UNASSIGNED
3289 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3290#endif
Richard Henderson5b450402011-04-18 16:13:12 -07003291#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
Blue Swirlb14ef7c2011-07-03 08:53:46 +00003292 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 2);
blueswir1e18231a2008-10-06 18:46:28 +00003293#endif
3294}
3295
Anthony Liguoric227f092009-10-01 16:12:16 -05003296static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
blueswir1e18231a2008-10-06 18:46:28 +00003297{
3298#ifdef DEBUG_UNASSIGNED
3299 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3300#endif
Richard Henderson5b450402011-04-18 16:13:12 -07003301#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
Blue Swirlb14ef7c2011-07-03 08:53:46 +00003302 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 4);
blueswir1b4f0a312007-05-06 17:59:24 +00003303#endif
bellard33417e72003-08-10 21:47:01 +00003304}
3305
Blue Swirld60efc62009-08-25 18:29:31 +00003306static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
bellard33417e72003-08-10 21:47:01 +00003307 unassigned_mem_readb,
blueswir1e18231a2008-10-06 18:46:28 +00003308 unassigned_mem_readw,
3309 unassigned_mem_readl,
bellard33417e72003-08-10 21:47:01 +00003310};
3311
Blue Swirld60efc62009-08-25 18:29:31 +00003312static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
bellard33417e72003-08-10 21:47:01 +00003313 unassigned_mem_writeb,
blueswir1e18231a2008-10-06 18:46:28 +00003314 unassigned_mem_writew,
3315 unassigned_mem_writel,
bellard33417e72003-08-10 21:47:01 +00003316};
3317
Anthony Liguoric227f092009-10-01 16:12:16 -05003318static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
pbrook0f459d12008-06-09 00:20:13 +00003319 uint32_t val)
bellard1ccde1c2004-02-06 19:46:14 +00003320{
bellard3a7d9292005-08-21 09:26:42 +00003321 int dirty_flags;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003322 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003323 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3324#if !defined(CONFIG_USER_ONLY)
3325 tb_invalidate_phys_page_fast(ram_addr, 1);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003326 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003327#endif
3328 }
pbrook5579c7f2009-04-11 14:47:08 +00003329 stb_p(qemu_get_ram_ptr(ram_addr), val);
bellardf23db162005-08-21 19:12:28 +00003330 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003331 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00003332 /* we remove the notdirty callback only if the code has been
3333 flushed */
3334 if (dirty_flags == 0xff)
pbrook2e70f6e2008-06-29 01:03:05 +00003335 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
bellard1ccde1c2004-02-06 19:46:14 +00003336}
3337
Anthony Liguoric227f092009-10-01 16:12:16 -05003338static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
pbrook0f459d12008-06-09 00:20:13 +00003339 uint32_t val)
bellard1ccde1c2004-02-06 19:46:14 +00003340{
bellard3a7d9292005-08-21 09:26:42 +00003341 int dirty_flags;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003342 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003343 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3344#if !defined(CONFIG_USER_ONLY)
3345 tb_invalidate_phys_page_fast(ram_addr, 2);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003346 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003347#endif
3348 }
pbrook5579c7f2009-04-11 14:47:08 +00003349 stw_p(qemu_get_ram_ptr(ram_addr), val);
bellardf23db162005-08-21 19:12:28 +00003350 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003351 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00003352 /* we remove the notdirty callback only if the code has been
3353 flushed */
3354 if (dirty_flags == 0xff)
pbrook2e70f6e2008-06-29 01:03:05 +00003355 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
bellard1ccde1c2004-02-06 19:46:14 +00003356}
3357
Anthony Liguoric227f092009-10-01 16:12:16 -05003358static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
pbrook0f459d12008-06-09 00:20:13 +00003359 uint32_t val)
bellard1ccde1c2004-02-06 19:46:14 +00003360{
bellard3a7d9292005-08-21 09:26:42 +00003361 int dirty_flags;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003362 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003363 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3364#if !defined(CONFIG_USER_ONLY)
3365 tb_invalidate_phys_page_fast(ram_addr, 4);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003366 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003367#endif
3368 }
pbrook5579c7f2009-04-11 14:47:08 +00003369 stl_p(qemu_get_ram_ptr(ram_addr), val);
bellardf23db162005-08-21 19:12:28 +00003370 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003371 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00003372 /* we remove the notdirty callback only if the code has been
3373 flushed */
3374 if (dirty_flags == 0xff)
pbrook2e70f6e2008-06-29 01:03:05 +00003375 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
bellard1ccde1c2004-02-06 19:46:14 +00003376}
3377
Blue Swirld60efc62009-08-25 18:29:31 +00003378static CPUReadMemoryFunc * const error_mem_read[3] = {
bellard3a7d9292005-08-21 09:26:42 +00003379 NULL, /* never used */
3380 NULL, /* never used */
3381 NULL, /* never used */
3382};
3383
Blue Swirld60efc62009-08-25 18:29:31 +00003384static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
bellard1ccde1c2004-02-06 19:46:14 +00003385 notdirty_mem_writeb,
3386 notdirty_mem_writew,
3387 notdirty_mem_writel,
3388};
3389
pbrook0f459d12008-06-09 00:20:13 +00003390/* Generate a debug exception if a watchpoint has been hit. */
aliguorib4051332008-11-18 20:14:20 +00003391static void check_watchpoint(int offset, int len_mask, int flags)
pbrook0f459d12008-06-09 00:20:13 +00003392{
3393 CPUState *env = cpu_single_env;
aliguori06d55cc2008-11-18 20:24:06 +00003394 target_ulong pc, cs_base;
3395 TranslationBlock *tb;
pbrook0f459d12008-06-09 00:20:13 +00003396 target_ulong vaddr;
aliguoria1d1bb32008-11-18 20:07:32 +00003397 CPUWatchpoint *wp;
aliguori06d55cc2008-11-18 20:24:06 +00003398 int cpu_flags;
pbrook0f459d12008-06-09 00:20:13 +00003399
aliguori06d55cc2008-11-18 20:24:06 +00003400 if (env->watchpoint_hit) {
3401 /* We re-entered the check after replacing the TB. Now raise
3402 * the debug interrupt so that is will trigger after the
3403 * current instruction. */
3404 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3405 return;
3406 }
pbrook2e70f6e2008-06-29 01:03:05 +00003407 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
Blue Swirl72cf2d42009-09-12 07:36:22 +00003408 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguorib4051332008-11-18 20:14:20 +00003409 if ((vaddr == (wp->vaddr & len_mask) ||
3410 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
aliguori6e140f22008-11-18 20:37:55 +00003411 wp->flags |= BP_WATCHPOINT_HIT;
3412 if (!env->watchpoint_hit) {
3413 env->watchpoint_hit = wp;
3414 tb = tb_find_pc(env->mem_io_pc);
3415 if (!tb) {
3416 cpu_abort(env, "check_watchpoint: could not find TB for "
3417 "pc=%p", (void *)env->mem_io_pc);
3418 }
Stefan Weil618ba8e2011-04-18 06:39:53 +00003419 cpu_restore_state(tb, env, env->mem_io_pc);
aliguori6e140f22008-11-18 20:37:55 +00003420 tb_phys_invalidate(tb, -1);
3421 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3422 env->exception_index = EXCP_DEBUG;
3423 } else {
3424 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3425 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3426 }
3427 cpu_resume_from_signal(env, NULL);
aliguori06d55cc2008-11-18 20:24:06 +00003428 }
aliguori6e140f22008-11-18 20:37:55 +00003429 } else {
3430 wp->flags &= ~BP_WATCHPOINT_HIT;
pbrook0f459d12008-06-09 00:20:13 +00003431 }
3432 }
3433}
3434
pbrook6658ffb2007-03-16 23:58:11 +00003435/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3436 so these check for a hit then pass through to the normal out-of-line
3437 phys routines. */
Anthony Liguoric227f092009-10-01 16:12:16 -05003438static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
pbrook6658ffb2007-03-16 23:58:11 +00003439{
aliguorib4051332008-11-18 20:14:20 +00003440 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
pbrook6658ffb2007-03-16 23:58:11 +00003441 return ldub_phys(addr);
3442}
3443
Anthony Liguoric227f092009-10-01 16:12:16 -05003444static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
pbrook6658ffb2007-03-16 23:58:11 +00003445{
aliguorib4051332008-11-18 20:14:20 +00003446 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
pbrook6658ffb2007-03-16 23:58:11 +00003447 return lduw_phys(addr);
3448}
3449
Anthony Liguoric227f092009-10-01 16:12:16 -05003450static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
pbrook6658ffb2007-03-16 23:58:11 +00003451{
aliguorib4051332008-11-18 20:14:20 +00003452 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
pbrook6658ffb2007-03-16 23:58:11 +00003453 return ldl_phys(addr);
3454}
3455
Anthony Liguoric227f092009-10-01 16:12:16 -05003456static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
pbrook6658ffb2007-03-16 23:58:11 +00003457 uint32_t val)
3458{
aliguorib4051332008-11-18 20:14:20 +00003459 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
pbrook6658ffb2007-03-16 23:58:11 +00003460 stb_phys(addr, val);
3461}
3462
Anthony Liguoric227f092009-10-01 16:12:16 -05003463static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
pbrook6658ffb2007-03-16 23:58:11 +00003464 uint32_t val)
3465{
aliguorib4051332008-11-18 20:14:20 +00003466 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
pbrook6658ffb2007-03-16 23:58:11 +00003467 stw_phys(addr, val);
3468}
3469
Anthony Liguoric227f092009-10-01 16:12:16 -05003470static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
pbrook6658ffb2007-03-16 23:58:11 +00003471 uint32_t val)
3472{
aliguorib4051332008-11-18 20:14:20 +00003473 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
pbrook6658ffb2007-03-16 23:58:11 +00003474 stl_phys(addr, val);
3475}
3476
Blue Swirld60efc62009-08-25 18:29:31 +00003477static CPUReadMemoryFunc * const watch_mem_read[3] = {
pbrook6658ffb2007-03-16 23:58:11 +00003478 watch_mem_readb,
3479 watch_mem_readw,
3480 watch_mem_readl,
3481};
3482
Blue Swirld60efc62009-08-25 18:29:31 +00003483static CPUWriteMemoryFunc * const watch_mem_write[3] = {
pbrook6658ffb2007-03-16 23:58:11 +00003484 watch_mem_writeb,
3485 watch_mem_writew,
3486 watch_mem_writel,
3487};
pbrook6658ffb2007-03-16 23:58:11 +00003488
Richard Hendersonf6405242010-04-22 16:47:31 -07003489static inline uint32_t subpage_readlen (subpage_t *mmio,
3490 target_phys_addr_t addr,
3491 unsigned int len)
blueswir1db7b5422007-05-26 17:36:03 +00003492{
Richard Hendersonf6405242010-04-22 16:47:31 -07003493 unsigned int idx = SUBPAGE_IDX(addr);
blueswir1db7b5422007-05-26 17:36:03 +00003494#if defined(DEBUG_SUBPAGE)
3495 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3496 mmio, len, addr, idx);
3497#endif
blueswir1db7b5422007-05-26 17:36:03 +00003498
Richard Hendersonf6405242010-04-22 16:47:31 -07003499 addr += mmio->region_offset[idx];
3500 idx = mmio->sub_io_index[idx];
3501 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
blueswir1db7b5422007-05-26 17:36:03 +00003502}
3503
Anthony Liguoric227f092009-10-01 16:12:16 -05003504static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
Richard Hendersonf6405242010-04-22 16:47:31 -07003505 uint32_t value, unsigned int len)
blueswir1db7b5422007-05-26 17:36:03 +00003506{
Richard Hendersonf6405242010-04-22 16:47:31 -07003507 unsigned int idx = SUBPAGE_IDX(addr);
blueswir1db7b5422007-05-26 17:36:03 +00003508#if defined(DEBUG_SUBPAGE)
Richard Hendersonf6405242010-04-22 16:47:31 -07003509 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3510 __func__, mmio, len, addr, idx, value);
blueswir1db7b5422007-05-26 17:36:03 +00003511#endif
Richard Hendersonf6405242010-04-22 16:47:31 -07003512
3513 addr += mmio->region_offset[idx];
3514 idx = mmio->sub_io_index[idx];
3515 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
blueswir1db7b5422007-05-26 17:36:03 +00003516}
3517
Anthony Liguoric227f092009-10-01 16:12:16 -05003518static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
blueswir1db7b5422007-05-26 17:36:03 +00003519{
blueswir1db7b5422007-05-26 17:36:03 +00003520 return subpage_readlen(opaque, addr, 0);
3521}
3522
Anthony Liguoric227f092009-10-01 16:12:16 -05003523static void subpage_writeb (void *opaque, target_phys_addr_t addr,
blueswir1db7b5422007-05-26 17:36:03 +00003524 uint32_t value)
3525{
blueswir1db7b5422007-05-26 17:36:03 +00003526 subpage_writelen(opaque, addr, value, 0);
3527}
3528
Anthony Liguoric227f092009-10-01 16:12:16 -05003529static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
blueswir1db7b5422007-05-26 17:36:03 +00003530{
blueswir1db7b5422007-05-26 17:36:03 +00003531 return subpage_readlen(opaque, addr, 1);
3532}
3533
Anthony Liguoric227f092009-10-01 16:12:16 -05003534static void subpage_writew (void *opaque, target_phys_addr_t addr,
blueswir1db7b5422007-05-26 17:36:03 +00003535 uint32_t value)
3536{
blueswir1db7b5422007-05-26 17:36:03 +00003537 subpage_writelen(opaque, addr, value, 1);
3538}
3539
Anthony Liguoric227f092009-10-01 16:12:16 -05003540static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
blueswir1db7b5422007-05-26 17:36:03 +00003541{
blueswir1db7b5422007-05-26 17:36:03 +00003542 return subpage_readlen(opaque, addr, 2);
3543}
3544
Richard Hendersonf6405242010-04-22 16:47:31 -07003545static void subpage_writel (void *opaque, target_phys_addr_t addr,
3546 uint32_t value)
blueswir1db7b5422007-05-26 17:36:03 +00003547{
blueswir1db7b5422007-05-26 17:36:03 +00003548 subpage_writelen(opaque, addr, value, 2);
3549}
3550
Blue Swirld60efc62009-08-25 18:29:31 +00003551static CPUReadMemoryFunc * const subpage_read[] = {
blueswir1db7b5422007-05-26 17:36:03 +00003552 &subpage_readb,
3553 &subpage_readw,
3554 &subpage_readl,
3555};
3556
Blue Swirld60efc62009-08-25 18:29:31 +00003557static CPUWriteMemoryFunc * const subpage_write[] = {
blueswir1db7b5422007-05-26 17:36:03 +00003558 &subpage_writeb,
3559 &subpage_writew,
3560 &subpage_writel,
3561};
3562
Anthony Liguoric227f092009-10-01 16:12:16 -05003563static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3564 ram_addr_t memory, ram_addr_t region_offset)
blueswir1db7b5422007-05-26 17:36:03 +00003565{
3566 int idx, eidx;
3567
3568 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3569 return -1;
3570 idx = SUBPAGE_IDX(start);
3571 eidx = SUBPAGE_IDX(end);
3572#if defined(DEBUG_SUBPAGE)
Blue Swirl0bf9e312009-07-20 17:19:25 +00003573 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
blueswir1db7b5422007-05-26 17:36:03 +00003574 mmio, start, end, idx, eidx, memory);
3575#endif
Gleb Natapov95c318f2010-07-29 10:41:45 +03003576 if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
3577 memory = IO_MEM_UNASSIGNED;
Richard Hendersonf6405242010-04-22 16:47:31 -07003578 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
blueswir1db7b5422007-05-26 17:36:03 +00003579 for (; idx <= eidx; idx++) {
Richard Hendersonf6405242010-04-22 16:47:31 -07003580 mmio->sub_io_index[idx] = memory;
3581 mmio->region_offset[idx] = region_offset;
blueswir1db7b5422007-05-26 17:36:03 +00003582 }
3583
3584 return 0;
3585}
3586
Richard Hendersonf6405242010-04-22 16:47:31 -07003587static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3588 ram_addr_t orig_memory,
3589 ram_addr_t region_offset)
blueswir1db7b5422007-05-26 17:36:03 +00003590{
Anthony Liguoric227f092009-10-01 16:12:16 -05003591 subpage_t *mmio;
blueswir1db7b5422007-05-26 17:36:03 +00003592 int subpage_memory;
3593
Anthony Liguoric227f092009-10-01 16:12:16 -05003594 mmio = qemu_mallocz(sizeof(subpage_t));
aliguori1eec6142009-02-05 22:06:18 +00003595
3596 mmio->base = base;
Alexander Graf2507c122010-12-08 12:05:37 +01003597 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
3598 DEVICE_NATIVE_ENDIAN);
blueswir1db7b5422007-05-26 17:36:03 +00003599#if defined(DEBUG_SUBPAGE)
aliguori1eec6142009-02-05 22:06:18 +00003600 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3601 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
blueswir1db7b5422007-05-26 17:36:03 +00003602#endif
aliguori1eec6142009-02-05 22:06:18 +00003603 *phys = subpage_memory | IO_MEM_SUBPAGE;
Richard Hendersonf6405242010-04-22 16:47:31 -07003604 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
blueswir1db7b5422007-05-26 17:36:03 +00003605
3606 return mmio;
3607}
3608
aliguori88715652009-02-11 15:20:58 +00003609static int get_free_io_mem_idx(void)
3610{
3611 int i;
3612
3613 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3614 if (!io_mem_used[i]) {
3615 io_mem_used[i] = 1;
3616 return i;
3617 }
Riku Voipioc6703b42009-12-03 15:56:05 +02003618 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
aliguori88715652009-02-11 15:20:58 +00003619 return -1;
3620}
3621
Alexander Grafdd310532010-12-08 12:05:36 +01003622/*
3623 * Usually, devices operate in little endian mode. There are devices out
3624 * there that operate in big endian too. Each device gets byte swapped
3625 * mmio if plugged onto a CPU that does the other endianness.
3626 *
3627 * CPU Device swap?
3628 *
3629 * little little no
3630 * little big yes
3631 * big little yes
3632 * big big no
3633 */
3634
3635typedef struct SwapEndianContainer {
3636 CPUReadMemoryFunc *read[3];
3637 CPUWriteMemoryFunc *write[3];
3638 void *opaque;
3639} SwapEndianContainer;
3640
3641static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
3642{
3643 uint32_t val;
3644 SwapEndianContainer *c = opaque;
3645 val = c->read[0](c->opaque, addr);
3646 return val;
3647}
3648
3649static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
3650{
3651 uint32_t val;
3652 SwapEndianContainer *c = opaque;
3653 val = bswap16(c->read[1](c->opaque, addr));
3654 return val;
3655}
3656
3657static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
3658{
3659 uint32_t val;
3660 SwapEndianContainer *c = opaque;
3661 val = bswap32(c->read[2](c->opaque, addr));
3662 return val;
3663}
3664
3665static CPUReadMemoryFunc * const swapendian_readfn[3]={
3666 swapendian_mem_readb,
3667 swapendian_mem_readw,
3668 swapendian_mem_readl
3669};
3670
3671static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
3672 uint32_t val)
3673{
3674 SwapEndianContainer *c = opaque;
3675 c->write[0](c->opaque, addr, val);
3676}
3677
3678static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
3679 uint32_t val)
3680{
3681 SwapEndianContainer *c = opaque;
3682 c->write[1](c->opaque, addr, bswap16(val));
3683}
3684
3685static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
3686 uint32_t val)
3687{
3688 SwapEndianContainer *c = opaque;
3689 c->write[2](c->opaque, addr, bswap32(val));
3690}
3691
3692static CPUWriteMemoryFunc * const swapendian_writefn[3]={
3693 swapendian_mem_writeb,
3694 swapendian_mem_writew,
3695 swapendian_mem_writel
3696};
3697
3698static void swapendian_init(int io_index)
3699{
3700 SwapEndianContainer *c = qemu_malloc(sizeof(SwapEndianContainer));
3701 int i;
3702
3703 /* Swap mmio for big endian targets */
3704 c->opaque = io_mem_opaque[io_index];
3705 for (i = 0; i < 3; i++) {
3706 c->read[i] = io_mem_read[io_index][i];
3707 c->write[i] = io_mem_write[io_index][i];
3708
3709 io_mem_read[io_index][i] = swapendian_readfn[i];
3710 io_mem_write[io_index][i] = swapendian_writefn[i];
3711 }
3712 io_mem_opaque[io_index] = c;
3713}
3714
3715static void swapendian_del(int io_index)
3716{
3717 if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
3718 qemu_free(io_mem_opaque[io_index]);
3719 }
3720}
3721
bellard33417e72003-08-10 21:47:01 +00003722/* mem_read and mem_write are arrays of functions containing the
3723 function to access byte (index 0), word (index 1) and dword (index
Paul Brook0b4e6e32009-04-30 18:37:55 +01003724 2). Functions can be omitted with a NULL function pointer.
blueswir13ee89922008-01-02 19:45:26 +00003725 If io_index is non zero, the corresponding io zone is
blueswir14254fab2008-01-01 16:57:19 +00003726 modified. If it is zero, a new io zone is allocated. The return
3727 value can be used with cpu_register_physical_memory(). (-1) is
3728 returned if error. */
Avi Kivity1eed09c2009-06-14 11:38:51 +03003729static int cpu_register_io_memory_fixed(int io_index,
Blue Swirld60efc62009-08-25 18:29:31 +00003730 CPUReadMemoryFunc * const *mem_read,
3731 CPUWriteMemoryFunc * const *mem_write,
Alexander Grafdd310532010-12-08 12:05:36 +01003732 void *opaque, enum device_endian endian)
bellard33417e72003-08-10 21:47:01 +00003733{
Richard Henderson3cab7212010-05-07 09:52:51 -07003734 int i;
3735
bellard33417e72003-08-10 21:47:01 +00003736 if (io_index <= 0) {
aliguori88715652009-02-11 15:20:58 +00003737 io_index = get_free_io_mem_idx();
3738 if (io_index == -1)
3739 return io_index;
bellard33417e72003-08-10 21:47:01 +00003740 } else {
Avi Kivity1eed09c2009-06-14 11:38:51 +03003741 io_index >>= IO_MEM_SHIFT;
bellard33417e72003-08-10 21:47:01 +00003742 if (io_index >= IO_MEM_NB_ENTRIES)
3743 return -1;
3744 }
bellardb5ff1b32005-11-26 10:38:39 +00003745
Richard Henderson3cab7212010-05-07 09:52:51 -07003746 for (i = 0; i < 3; ++i) {
3747 io_mem_read[io_index][i]
3748 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3749 }
3750 for (i = 0; i < 3; ++i) {
3751 io_mem_write[io_index][i]
3752 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3753 }
bellarda4193c82004-06-03 14:01:43 +00003754 io_mem_opaque[io_index] = opaque;
Richard Hendersonf6405242010-04-22 16:47:31 -07003755
Alexander Grafdd310532010-12-08 12:05:36 +01003756 switch (endian) {
3757 case DEVICE_BIG_ENDIAN:
3758#ifndef TARGET_WORDS_BIGENDIAN
3759 swapendian_init(io_index);
3760#endif
3761 break;
3762 case DEVICE_LITTLE_ENDIAN:
3763#ifdef TARGET_WORDS_BIGENDIAN
3764 swapendian_init(io_index);
3765#endif
3766 break;
3767 case DEVICE_NATIVE_ENDIAN:
3768 default:
3769 break;
3770 }
3771
Richard Hendersonf6405242010-04-22 16:47:31 -07003772 return (io_index << IO_MEM_SHIFT);
bellard33417e72003-08-10 21:47:01 +00003773}
bellard61382a52003-10-27 21:22:23 +00003774
Blue Swirld60efc62009-08-25 18:29:31 +00003775int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3776 CPUWriteMemoryFunc * const *mem_write,
Alexander Grafdd310532010-12-08 12:05:36 +01003777 void *opaque, enum device_endian endian)
Avi Kivity1eed09c2009-06-14 11:38:51 +03003778{
Alexander Graf2507c122010-12-08 12:05:37 +01003779 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
Avi Kivity1eed09c2009-06-14 11:38:51 +03003780}
3781
aliguori88715652009-02-11 15:20:58 +00003782void cpu_unregister_io_memory(int io_table_address)
3783{
3784 int i;
3785 int io_index = io_table_address >> IO_MEM_SHIFT;
3786
Alexander Grafdd310532010-12-08 12:05:36 +01003787 swapendian_del(io_index);
3788
aliguori88715652009-02-11 15:20:58 +00003789 for (i=0;i < 3; i++) {
3790 io_mem_read[io_index][i] = unassigned_mem_read[i];
3791 io_mem_write[io_index][i] = unassigned_mem_write[i];
3792 }
3793 io_mem_opaque[io_index] = NULL;
3794 io_mem_used[io_index] = 0;
3795}
3796
Avi Kivitye9179ce2009-06-14 11:38:52 +03003797static void io_mem_init(void)
3798{
3799 int i;
3800
Alexander Graf2507c122010-12-08 12:05:37 +01003801 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
3802 unassigned_mem_write, NULL,
3803 DEVICE_NATIVE_ENDIAN);
3804 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
3805 unassigned_mem_write, NULL,
3806 DEVICE_NATIVE_ENDIAN);
3807 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
3808 notdirty_mem_write, NULL,
3809 DEVICE_NATIVE_ENDIAN);
Avi Kivitye9179ce2009-06-14 11:38:52 +03003810 for (i=0; i<5; i++)
3811 io_mem_used[i] = 1;
3812
3813 io_mem_watch = cpu_register_io_memory(watch_mem_read,
Alexander Graf2507c122010-12-08 12:05:37 +01003814 watch_mem_write, NULL,
3815 DEVICE_NATIVE_ENDIAN);
Avi Kivitye9179ce2009-06-14 11:38:52 +03003816}
3817
Avi Kivity62152b82011-07-26 14:26:14 +03003818static void memory_map_init(void)
3819{
3820 system_memory = qemu_malloc(sizeof(*system_memory));
Avi Kivity8417ceb2011-08-03 11:56:14 +03003821 memory_region_init(system_memory, "system", INT64_MAX);
Avi Kivity62152b82011-07-26 14:26:14 +03003822 set_system_memory_map(system_memory);
3823}
3824
3825MemoryRegion *get_system_memory(void)
3826{
3827 return system_memory;
3828}
3829
pbrooke2eef172008-06-08 01:09:01 +00003830#endif /* !defined(CONFIG_USER_ONLY) */
3831
bellard13eb76e2004-01-24 15:23:36 +00003832/* physical memory access (slow version, mainly for debug) */
3833#if defined(CONFIG_USER_ONLY)
Paul Brooka68fe892010-03-01 00:08:59 +00003834int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3835 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00003836{
3837 int l, flags;
3838 target_ulong page;
pbrook53a59602006-03-25 19:31:22 +00003839 void * p;
bellard13eb76e2004-01-24 15:23:36 +00003840
3841 while (len > 0) {
3842 page = addr & TARGET_PAGE_MASK;
3843 l = (page + TARGET_PAGE_SIZE) - addr;
3844 if (l > len)
3845 l = len;
3846 flags = page_get_flags(page);
3847 if (!(flags & PAGE_VALID))
Paul Brooka68fe892010-03-01 00:08:59 +00003848 return -1;
bellard13eb76e2004-01-24 15:23:36 +00003849 if (is_write) {
3850 if (!(flags & PAGE_WRITE))
Paul Brooka68fe892010-03-01 00:08:59 +00003851 return -1;
bellard579a97f2007-11-11 14:26:47 +00003852 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00003853 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
Paul Brooka68fe892010-03-01 00:08:59 +00003854 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00003855 memcpy(p, buf, l);
3856 unlock_user(p, addr, l);
bellard13eb76e2004-01-24 15:23:36 +00003857 } else {
3858 if (!(flags & PAGE_READ))
Paul Brooka68fe892010-03-01 00:08:59 +00003859 return -1;
bellard579a97f2007-11-11 14:26:47 +00003860 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00003861 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
Paul Brooka68fe892010-03-01 00:08:59 +00003862 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00003863 memcpy(buf, p, l);
aurel325b257572008-04-28 08:54:59 +00003864 unlock_user(p, addr, 0);
bellard13eb76e2004-01-24 15:23:36 +00003865 }
3866 len -= l;
3867 buf += l;
3868 addr += l;
3869 }
Paul Brooka68fe892010-03-01 00:08:59 +00003870 return 0;
bellard13eb76e2004-01-24 15:23:36 +00003871}
bellard8df1cd02005-01-28 22:37:22 +00003872
bellard13eb76e2004-01-24 15:23:36 +00003873#else
Anthony Liguoric227f092009-10-01 16:12:16 -05003874void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
bellard13eb76e2004-01-24 15:23:36 +00003875 int len, int is_write)
3876{
3877 int l, io_index;
3878 uint8_t *ptr;
3879 uint32_t val;
Anthony Liguoric227f092009-10-01 16:12:16 -05003880 target_phys_addr_t page;
Anthony PERARD8ca56922011-07-15 04:32:53 +00003881 ram_addr_t pd;
bellard92e873b2004-05-21 14:52:29 +00003882 PhysPageDesc *p;
ths3b46e622007-09-17 08:09:54 +00003883
bellard13eb76e2004-01-24 15:23:36 +00003884 while (len > 0) {
3885 page = addr & TARGET_PAGE_MASK;
3886 l = (page + TARGET_PAGE_SIZE) - addr;
3887 if (l > len)
3888 l = len;
bellard92e873b2004-05-21 14:52:29 +00003889 p = phys_page_find(page >> TARGET_PAGE_BITS);
bellard13eb76e2004-01-24 15:23:36 +00003890 if (!p) {
3891 pd = IO_MEM_UNASSIGNED;
3892 } else {
3893 pd = p->phys_offset;
3894 }
ths3b46e622007-09-17 08:09:54 +00003895
bellard13eb76e2004-01-24 15:23:36 +00003896 if (is_write) {
bellard3a7d9292005-08-21 09:26:42 +00003897 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
Anthony Liguoric227f092009-10-01 16:12:16 -05003898 target_phys_addr_t addr1 = addr;
bellard13eb76e2004-01-24 15:23:36 +00003899 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00003900 if (p)
aurel326c2934d2009-02-18 21:37:17 +00003901 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard6a00d602005-11-21 23:25:50 +00003902 /* XXX: could force cpu_single_env to NULL to avoid
3903 potential bugs */
aurel326c2934d2009-02-18 21:37:17 +00003904 if (l >= 4 && ((addr1 & 3) == 0)) {
bellard1c213d12005-09-03 10:49:04 +00003905 /* 32 bit write access */
bellardc27004e2005-01-03 23:35:10 +00003906 val = ldl_p(buf);
aurel326c2934d2009-02-18 21:37:17 +00003907 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
bellard13eb76e2004-01-24 15:23:36 +00003908 l = 4;
aurel326c2934d2009-02-18 21:37:17 +00003909 } else if (l >= 2 && ((addr1 & 1) == 0)) {
bellard1c213d12005-09-03 10:49:04 +00003910 /* 16 bit write access */
bellardc27004e2005-01-03 23:35:10 +00003911 val = lduw_p(buf);
aurel326c2934d2009-02-18 21:37:17 +00003912 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
bellard13eb76e2004-01-24 15:23:36 +00003913 l = 2;
3914 } else {
bellard1c213d12005-09-03 10:49:04 +00003915 /* 8 bit write access */
bellardc27004e2005-01-03 23:35:10 +00003916 val = ldub_p(buf);
aurel326c2934d2009-02-18 21:37:17 +00003917 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
bellard13eb76e2004-01-24 15:23:36 +00003918 l = 1;
3919 }
3920 } else {
Anthony PERARD8ca56922011-07-15 04:32:53 +00003921 ram_addr_t addr1;
bellardb448f2f2004-02-25 23:24:04 +00003922 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
bellard13eb76e2004-01-24 15:23:36 +00003923 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00003924 ptr = qemu_get_ram_ptr(addr1);
bellard13eb76e2004-01-24 15:23:36 +00003925 memcpy(ptr, buf, l);
bellard3a7d9292005-08-21 09:26:42 +00003926 if (!cpu_physical_memory_is_dirty(addr1)) {
3927 /* invalidate code */
3928 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3929 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003930 cpu_physical_memory_set_dirty_flags(
3931 addr1, (0xff & ~CODE_DIRTY_FLAG));
bellard3a7d9292005-08-21 09:26:42 +00003932 }
Anthony PERARD050a0dd2010-09-16 13:57:49 +01003933 qemu_put_ram_ptr(ptr);
bellard13eb76e2004-01-24 15:23:36 +00003934 }
3935 } else {
ths5fafdf22007-09-16 21:08:06 +00003936 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
bellard2a4188a2006-06-25 21:54:59 +00003937 !(pd & IO_MEM_ROMD)) {
Anthony Liguoric227f092009-10-01 16:12:16 -05003938 target_phys_addr_t addr1 = addr;
bellard13eb76e2004-01-24 15:23:36 +00003939 /* I/O case */
3940 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00003941 if (p)
aurel326c2934d2009-02-18 21:37:17 +00003942 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3943 if (l >= 4 && ((addr1 & 3) == 0)) {
bellard13eb76e2004-01-24 15:23:36 +00003944 /* 32 bit read access */
aurel326c2934d2009-02-18 21:37:17 +00003945 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
bellardc27004e2005-01-03 23:35:10 +00003946 stl_p(buf, val);
bellard13eb76e2004-01-24 15:23:36 +00003947 l = 4;
aurel326c2934d2009-02-18 21:37:17 +00003948 } else if (l >= 2 && ((addr1 & 1) == 0)) {
bellard13eb76e2004-01-24 15:23:36 +00003949 /* 16 bit read access */
aurel326c2934d2009-02-18 21:37:17 +00003950 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
bellardc27004e2005-01-03 23:35:10 +00003951 stw_p(buf, val);
bellard13eb76e2004-01-24 15:23:36 +00003952 l = 2;
3953 } else {
bellard1c213d12005-09-03 10:49:04 +00003954 /* 8 bit read access */
aurel326c2934d2009-02-18 21:37:17 +00003955 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
bellardc27004e2005-01-03 23:35:10 +00003956 stb_p(buf, val);
bellard13eb76e2004-01-24 15:23:36 +00003957 l = 1;
3958 }
3959 } else {
3960 /* RAM case */
Anthony PERARD050a0dd2010-09-16 13:57:49 +01003961 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
3962 memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l);
3963 qemu_put_ram_ptr(ptr);
bellard13eb76e2004-01-24 15:23:36 +00003964 }
3965 }
3966 len -= l;
3967 buf += l;
3968 addr += l;
3969 }
3970}
bellard8df1cd02005-01-28 22:37:22 +00003971
bellardd0ecd2a2006-04-23 17:14:48 +00003972/* used for ROM loading : can write in RAM and ROM */
Anthony Liguoric227f092009-10-01 16:12:16 -05003973void cpu_physical_memory_write_rom(target_phys_addr_t addr,
bellardd0ecd2a2006-04-23 17:14:48 +00003974 const uint8_t *buf, int len)
3975{
3976 int l;
3977 uint8_t *ptr;
Anthony Liguoric227f092009-10-01 16:12:16 -05003978 target_phys_addr_t page;
bellardd0ecd2a2006-04-23 17:14:48 +00003979 unsigned long pd;
3980 PhysPageDesc *p;
ths3b46e622007-09-17 08:09:54 +00003981
bellardd0ecd2a2006-04-23 17:14:48 +00003982 while (len > 0) {
3983 page = addr & TARGET_PAGE_MASK;
3984 l = (page + TARGET_PAGE_SIZE) - addr;
3985 if (l > len)
3986 l = len;
3987 p = phys_page_find(page >> TARGET_PAGE_BITS);
3988 if (!p) {
3989 pd = IO_MEM_UNASSIGNED;
3990 } else {
3991 pd = p->phys_offset;
3992 }
ths3b46e622007-09-17 08:09:54 +00003993
bellardd0ecd2a2006-04-23 17:14:48 +00003994 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
bellard2a4188a2006-06-25 21:54:59 +00003995 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3996 !(pd & IO_MEM_ROMD)) {
bellardd0ecd2a2006-04-23 17:14:48 +00003997 /* do nothing */
3998 } else {
3999 unsigned long addr1;
4000 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4001 /* ROM/RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00004002 ptr = qemu_get_ram_ptr(addr1);
bellardd0ecd2a2006-04-23 17:14:48 +00004003 memcpy(ptr, buf, l);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01004004 qemu_put_ram_ptr(ptr);
bellardd0ecd2a2006-04-23 17:14:48 +00004005 }
4006 len -= l;
4007 buf += l;
4008 addr += l;
4009 }
4010}
4011
aliguori6d16c2f2009-01-22 16:59:11 +00004012typedef struct {
4013 void *buffer;
Anthony Liguoric227f092009-10-01 16:12:16 -05004014 target_phys_addr_t addr;
4015 target_phys_addr_t len;
aliguori6d16c2f2009-01-22 16:59:11 +00004016} BounceBuffer;
4017
4018static BounceBuffer bounce;
4019
aliguoriba223c22009-01-22 16:59:16 +00004020typedef struct MapClient {
4021 void *opaque;
4022 void (*callback)(void *opaque);
Blue Swirl72cf2d42009-09-12 07:36:22 +00004023 QLIST_ENTRY(MapClient) link;
aliguoriba223c22009-01-22 16:59:16 +00004024} MapClient;
4025
Blue Swirl72cf2d42009-09-12 07:36:22 +00004026static QLIST_HEAD(map_client_list, MapClient) map_client_list
4027 = QLIST_HEAD_INITIALIZER(map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00004028
4029void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
4030{
4031 MapClient *client = qemu_malloc(sizeof(*client));
4032
4033 client->opaque = opaque;
4034 client->callback = callback;
Blue Swirl72cf2d42009-09-12 07:36:22 +00004035 QLIST_INSERT_HEAD(&map_client_list, client, link);
aliguoriba223c22009-01-22 16:59:16 +00004036 return client;
4037}
4038
4039void cpu_unregister_map_client(void *_client)
4040{
4041 MapClient *client = (MapClient *)_client;
4042
Blue Swirl72cf2d42009-09-12 07:36:22 +00004043 QLIST_REMOVE(client, link);
Isaku Yamahata34d5e942009-06-26 18:57:18 +09004044 qemu_free(client);
aliguoriba223c22009-01-22 16:59:16 +00004045}
4046
4047static void cpu_notify_map_clients(void)
4048{
4049 MapClient *client;
4050
Blue Swirl72cf2d42009-09-12 07:36:22 +00004051 while (!QLIST_EMPTY(&map_client_list)) {
4052 client = QLIST_FIRST(&map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00004053 client->callback(client->opaque);
Isaku Yamahata34d5e942009-06-26 18:57:18 +09004054 cpu_unregister_map_client(client);
aliguoriba223c22009-01-22 16:59:16 +00004055 }
4056}
4057
aliguori6d16c2f2009-01-22 16:59:11 +00004058/* Map a physical memory region into a host virtual address.
4059 * May map a subset of the requested range, given by and returned in *plen.
4060 * May return NULL if resources needed to perform the mapping are exhausted.
4061 * Use only for reads OR writes - not for read-modify-write operations.
aliguoriba223c22009-01-22 16:59:16 +00004062 * Use cpu_register_map_client() to know when retrying the map operation is
4063 * likely to succeed.
aliguori6d16c2f2009-01-22 16:59:11 +00004064 */
Anthony Liguoric227f092009-10-01 16:12:16 -05004065void *cpu_physical_memory_map(target_phys_addr_t addr,
4066 target_phys_addr_t *plen,
aliguori6d16c2f2009-01-22 16:59:11 +00004067 int is_write)
4068{
Anthony Liguoric227f092009-10-01 16:12:16 -05004069 target_phys_addr_t len = *plen;
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01004070 target_phys_addr_t todo = 0;
aliguori6d16c2f2009-01-22 16:59:11 +00004071 int l;
Anthony Liguoric227f092009-10-01 16:12:16 -05004072 target_phys_addr_t page;
aliguori6d16c2f2009-01-22 16:59:11 +00004073 unsigned long pd;
4074 PhysPageDesc *p;
Anthony PERARDf15fbc42011-07-20 08:17:42 +00004075 ram_addr_t raddr = RAM_ADDR_MAX;
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01004076 ram_addr_t rlen;
4077 void *ret;
aliguori6d16c2f2009-01-22 16:59:11 +00004078
4079 while (len > 0) {
4080 page = addr & TARGET_PAGE_MASK;
4081 l = (page + TARGET_PAGE_SIZE) - addr;
4082 if (l > len)
4083 l = len;
4084 p = phys_page_find(page >> TARGET_PAGE_BITS);
4085 if (!p) {
4086 pd = IO_MEM_UNASSIGNED;
4087 } else {
4088 pd = p->phys_offset;
4089 }
4090
4091 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01004092 if (todo || bounce.buffer) {
aliguori6d16c2f2009-01-22 16:59:11 +00004093 break;
4094 }
4095 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
4096 bounce.addr = addr;
4097 bounce.len = l;
4098 if (!is_write) {
Stefan Weil54f7b4a2011-04-10 18:23:39 +02004099 cpu_physical_memory_read(addr, bounce.buffer, l);
aliguori6d16c2f2009-01-22 16:59:11 +00004100 }
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01004101
4102 *plen = l;
4103 return bounce.buffer;
aliguori6d16c2f2009-01-22 16:59:11 +00004104 }
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01004105 if (!todo) {
4106 raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4107 }
aliguori6d16c2f2009-01-22 16:59:11 +00004108
4109 len -= l;
4110 addr += l;
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01004111 todo += l;
aliguori6d16c2f2009-01-22 16:59:11 +00004112 }
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01004113 rlen = todo;
4114 ret = qemu_ram_ptr_length(raddr, &rlen);
4115 *plen = rlen;
4116 return ret;
aliguori6d16c2f2009-01-22 16:59:11 +00004117}
4118
4119/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
4120 * Will also mark the memory as dirty if is_write == 1. access_len gives
4121 * the amount of memory that was actually read or written by the caller.
4122 */
Anthony Liguoric227f092009-10-01 16:12:16 -05004123void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
4124 int is_write, target_phys_addr_t access_len)
aliguori6d16c2f2009-01-22 16:59:11 +00004125{
4126 if (buffer != bounce.buffer) {
4127 if (is_write) {
Marcelo Tosattie8902612010-10-11 15:31:19 -03004128 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00004129 while (access_len) {
4130 unsigned l;
4131 l = TARGET_PAGE_SIZE;
4132 if (l > access_len)
4133 l = access_len;
4134 if (!cpu_physical_memory_is_dirty(addr1)) {
4135 /* invalidate code */
4136 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
4137 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09004138 cpu_physical_memory_set_dirty_flags(
4139 addr1, (0xff & ~CODE_DIRTY_FLAG));
aliguori6d16c2f2009-01-22 16:59:11 +00004140 }
4141 addr1 += l;
4142 access_len -= l;
4143 }
4144 }
Jan Kiszka868bb332011-06-21 22:59:09 +02004145 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02004146 xen_invalidate_map_cache_entry(buffer);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01004147 }
aliguori6d16c2f2009-01-22 16:59:11 +00004148 return;
4149 }
4150 if (is_write) {
4151 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
4152 }
Herve Poussineauf8a83242010-01-24 21:23:56 +00004153 qemu_vfree(bounce.buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00004154 bounce.buffer = NULL;
aliguoriba223c22009-01-22 16:59:16 +00004155 cpu_notify_map_clients();
aliguori6d16c2f2009-01-22 16:59:11 +00004156}
bellardd0ecd2a2006-04-23 17:14:48 +00004157
bellard8df1cd02005-01-28 22:37:22 +00004158/* warning: addr must be aligned */
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004159static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
4160 enum device_endian endian)
bellard8df1cd02005-01-28 22:37:22 +00004161{
4162 int io_index;
4163 uint8_t *ptr;
4164 uint32_t val;
4165 unsigned long pd;
4166 PhysPageDesc *p;
4167
4168 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4169 if (!p) {
4170 pd = IO_MEM_UNASSIGNED;
4171 } else {
4172 pd = p->phys_offset;
4173 }
ths3b46e622007-09-17 08:09:54 +00004174
ths5fafdf22007-09-16 21:08:06 +00004175 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
bellard2a4188a2006-06-25 21:54:59 +00004176 !(pd & IO_MEM_ROMD)) {
bellard8df1cd02005-01-28 22:37:22 +00004177 /* I/O case */
4178 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004179 if (p)
4180 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard8df1cd02005-01-28 22:37:22 +00004181 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004182#if defined(TARGET_WORDS_BIGENDIAN)
4183 if (endian == DEVICE_LITTLE_ENDIAN) {
4184 val = bswap32(val);
4185 }
4186#else
4187 if (endian == DEVICE_BIG_ENDIAN) {
4188 val = bswap32(val);
4189 }
4190#endif
bellard8df1cd02005-01-28 22:37:22 +00004191 } else {
4192 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00004193 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
bellard8df1cd02005-01-28 22:37:22 +00004194 (addr & ~TARGET_PAGE_MASK);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004195 switch (endian) {
4196 case DEVICE_LITTLE_ENDIAN:
4197 val = ldl_le_p(ptr);
4198 break;
4199 case DEVICE_BIG_ENDIAN:
4200 val = ldl_be_p(ptr);
4201 break;
4202 default:
4203 val = ldl_p(ptr);
4204 break;
4205 }
bellard8df1cd02005-01-28 22:37:22 +00004206 }
4207 return val;
4208}
4209
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004210uint32_t ldl_phys(target_phys_addr_t addr)
4211{
4212 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
4213}
4214
4215uint32_t ldl_le_phys(target_phys_addr_t addr)
4216{
4217 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
4218}
4219
4220uint32_t ldl_be_phys(target_phys_addr_t addr)
4221{
4222 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
4223}
4224
bellard84b7b8e2005-11-28 21:19:04 +00004225/* warning: addr must be aligned */
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004226static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
4227 enum device_endian endian)
bellard84b7b8e2005-11-28 21:19:04 +00004228{
4229 int io_index;
4230 uint8_t *ptr;
4231 uint64_t val;
4232 unsigned long pd;
4233 PhysPageDesc *p;
4234
4235 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4236 if (!p) {
4237 pd = IO_MEM_UNASSIGNED;
4238 } else {
4239 pd = p->phys_offset;
4240 }
ths3b46e622007-09-17 08:09:54 +00004241
bellard2a4188a2006-06-25 21:54:59 +00004242 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4243 !(pd & IO_MEM_ROMD)) {
bellard84b7b8e2005-11-28 21:19:04 +00004244 /* I/O case */
4245 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004246 if (p)
4247 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004248
4249 /* XXX This is broken when device endian != cpu endian.
4250 Fix and add "endian" variable check */
bellard84b7b8e2005-11-28 21:19:04 +00004251#ifdef TARGET_WORDS_BIGENDIAN
4252 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
4253 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
4254#else
4255 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4256 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
4257#endif
4258 } else {
4259 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00004260 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
bellard84b7b8e2005-11-28 21:19:04 +00004261 (addr & ~TARGET_PAGE_MASK);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004262 switch (endian) {
4263 case DEVICE_LITTLE_ENDIAN:
4264 val = ldq_le_p(ptr);
4265 break;
4266 case DEVICE_BIG_ENDIAN:
4267 val = ldq_be_p(ptr);
4268 break;
4269 default:
4270 val = ldq_p(ptr);
4271 break;
4272 }
bellard84b7b8e2005-11-28 21:19:04 +00004273 }
4274 return val;
4275}
4276
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004277uint64_t ldq_phys(target_phys_addr_t addr)
4278{
4279 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
4280}
4281
4282uint64_t ldq_le_phys(target_phys_addr_t addr)
4283{
4284 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
4285}
4286
4287uint64_t ldq_be_phys(target_phys_addr_t addr)
4288{
4289 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
4290}
4291
bellardaab33092005-10-30 20:48:42 +00004292/* XXX: optimize */
Anthony Liguoric227f092009-10-01 16:12:16 -05004293uint32_t ldub_phys(target_phys_addr_t addr)
bellardaab33092005-10-30 20:48:42 +00004294{
4295 uint8_t val;
4296 cpu_physical_memory_read(addr, &val, 1);
4297 return val;
4298}
4299
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004300/* warning: addr must be aligned */
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004301static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
4302 enum device_endian endian)
bellardaab33092005-10-30 20:48:42 +00004303{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004304 int io_index;
4305 uint8_t *ptr;
4306 uint64_t val;
4307 unsigned long pd;
4308 PhysPageDesc *p;
4309
4310 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4311 if (!p) {
4312 pd = IO_MEM_UNASSIGNED;
4313 } else {
4314 pd = p->phys_offset;
4315 }
4316
4317 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4318 !(pd & IO_MEM_ROMD)) {
4319 /* I/O case */
4320 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4321 if (p)
4322 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4323 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004324#if defined(TARGET_WORDS_BIGENDIAN)
4325 if (endian == DEVICE_LITTLE_ENDIAN) {
4326 val = bswap16(val);
4327 }
4328#else
4329 if (endian == DEVICE_BIG_ENDIAN) {
4330 val = bswap16(val);
4331 }
4332#endif
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004333 } else {
4334 /* RAM case */
4335 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4336 (addr & ~TARGET_PAGE_MASK);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004337 switch (endian) {
4338 case DEVICE_LITTLE_ENDIAN:
4339 val = lduw_le_p(ptr);
4340 break;
4341 case DEVICE_BIG_ENDIAN:
4342 val = lduw_be_p(ptr);
4343 break;
4344 default:
4345 val = lduw_p(ptr);
4346 break;
4347 }
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004348 }
4349 return val;
bellardaab33092005-10-30 20:48:42 +00004350}
4351
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004352uint32_t lduw_phys(target_phys_addr_t addr)
4353{
4354 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
4355}
4356
4357uint32_t lduw_le_phys(target_phys_addr_t addr)
4358{
4359 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
4360}
4361
4362uint32_t lduw_be_phys(target_phys_addr_t addr)
4363{
4364 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
4365}
4366
bellard8df1cd02005-01-28 22:37:22 +00004367/* warning: addr must be aligned. The ram page is not masked as dirty
4368 and the code inside is not invalidated. It is useful if the dirty
4369 bits are used to track modified PTEs */
Anthony Liguoric227f092009-10-01 16:12:16 -05004370void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
bellard8df1cd02005-01-28 22:37:22 +00004371{
4372 int io_index;
4373 uint8_t *ptr;
4374 unsigned long pd;
4375 PhysPageDesc *p;
4376
4377 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4378 if (!p) {
4379 pd = IO_MEM_UNASSIGNED;
4380 } else {
4381 pd = p->phys_offset;
4382 }
ths3b46e622007-09-17 08:09:54 +00004383
bellard3a7d9292005-08-21 09:26:42 +00004384 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
bellard8df1cd02005-01-28 22:37:22 +00004385 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004386 if (p)
4387 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard8df1cd02005-01-28 22:37:22 +00004388 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4389 } else {
aliguori74576192008-10-06 14:02:03 +00004390 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
pbrook5579c7f2009-04-11 14:47:08 +00004391 ptr = qemu_get_ram_ptr(addr1);
bellard8df1cd02005-01-28 22:37:22 +00004392 stl_p(ptr, val);
aliguori74576192008-10-06 14:02:03 +00004393
4394 if (unlikely(in_migration)) {
4395 if (!cpu_physical_memory_is_dirty(addr1)) {
4396 /* invalidate code */
4397 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4398 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09004399 cpu_physical_memory_set_dirty_flags(
4400 addr1, (0xff & ~CODE_DIRTY_FLAG));
aliguori74576192008-10-06 14:02:03 +00004401 }
4402 }
bellard8df1cd02005-01-28 22:37:22 +00004403 }
4404}
4405
Anthony Liguoric227f092009-10-01 16:12:16 -05004406void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
j_mayerbc98a7e2007-04-04 07:55:12 +00004407{
4408 int io_index;
4409 uint8_t *ptr;
4410 unsigned long pd;
4411 PhysPageDesc *p;
4412
4413 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4414 if (!p) {
4415 pd = IO_MEM_UNASSIGNED;
4416 } else {
4417 pd = p->phys_offset;
4418 }
ths3b46e622007-09-17 08:09:54 +00004419
j_mayerbc98a7e2007-04-04 07:55:12 +00004420 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4421 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004422 if (p)
4423 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
j_mayerbc98a7e2007-04-04 07:55:12 +00004424#ifdef TARGET_WORDS_BIGENDIAN
4425 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4426 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4427#else
4428 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4429 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4430#endif
4431 } else {
pbrook5579c7f2009-04-11 14:47:08 +00004432 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
j_mayerbc98a7e2007-04-04 07:55:12 +00004433 (addr & ~TARGET_PAGE_MASK);
4434 stq_p(ptr, val);
4435 }
4436}
4437
bellard8df1cd02005-01-28 22:37:22 +00004438/* warning: addr must be aligned */
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004439static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
4440 enum device_endian endian)
bellard8df1cd02005-01-28 22:37:22 +00004441{
4442 int io_index;
4443 uint8_t *ptr;
4444 unsigned long pd;
4445 PhysPageDesc *p;
4446
4447 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4448 if (!p) {
4449 pd = IO_MEM_UNASSIGNED;
4450 } else {
4451 pd = p->phys_offset;
4452 }
ths3b46e622007-09-17 08:09:54 +00004453
bellard3a7d9292005-08-21 09:26:42 +00004454 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
bellard8df1cd02005-01-28 22:37:22 +00004455 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004456 if (p)
4457 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004458#if defined(TARGET_WORDS_BIGENDIAN)
4459 if (endian == DEVICE_LITTLE_ENDIAN) {
4460 val = bswap32(val);
4461 }
4462#else
4463 if (endian == DEVICE_BIG_ENDIAN) {
4464 val = bswap32(val);
4465 }
4466#endif
bellard8df1cd02005-01-28 22:37:22 +00004467 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4468 } else {
4469 unsigned long addr1;
4470 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4471 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00004472 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004473 switch (endian) {
4474 case DEVICE_LITTLE_ENDIAN:
4475 stl_le_p(ptr, val);
4476 break;
4477 case DEVICE_BIG_ENDIAN:
4478 stl_be_p(ptr, val);
4479 break;
4480 default:
4481 stl_p(ptr, val);
4482 break;
4483 }
bellard3a7d9292005-08-21 09:26:42 +00004484 if (!cpu_physical_memory_is_dirty(addr1)) {
4485 /* invalidate code */
4486 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4487 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09004488 cpu_physical_memory_set_dirty_flags(addr1,
4489 (0xff & ~CODE_DIRTY_FLAG));
bellard3a7d9292005-08-21 09:26:42 +00004490 }
bellard8df1cd02005-01-28 22:37:22 +00004491 }
4492}
4493
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004494void stl_phys(target_phys_addr_t addr, uint32_t val)
4495{
4496 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
4497}
4498
4499void stl_le_phys(target_phys_addr_t addr, uint32_t val)
4500{
4501 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
4502}
4503
4504void stl_be_phys(target_phys_addr_t addr, uint32_t val)
4505{
4506 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
4507}
4508
bellardaab33092005-10-30 20:48:42 +00004509/* XXX: optimize */
Anthony Liguoric227f092009-10-01 16:12:16 -05004510void stb_phys(target_phys_addr_t addr, uint32_t val)
bellardaab33092005-10-30 20:48:42 +00004511{
4512 uint8_t v = val;
4513 cpu_physical_memory_write(addr, &v, 1);
4514}
4515
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004516/* warning: addr must be aligned */
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004517static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
4518 enum device_endian endian)
bellardaab33092005-10-30 20:48:42 +00004519{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004520 int io_index;
4521 uint8_t *ptr;
4522 unsigned long pd;
4523 PhysPageDesc *p;
4524
4525 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4526 if (!p) {
4527 pd = IO_MEM_UNASSIGNED;
4528 } else {
4529 pd = p->phys_offset;
4530 }
4531
4532 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4533 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4534 if (p)
4535 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004536#if defined(TARGET_WORDS_BIGENDIAN)
4537 if (endian == DEVICE_LITTLE_ENDIAN) {
4538 val = bswap16(val);
4539 }
4540#else
4541 if (endian == DEVICE_BIG_ENDIAN) {
4542 val = bswap16(val);
4543 }
4544#endif
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004545 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
4546 } else {
4547 unsigned long addr1;
4548 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4549 /* RAM case */
4550 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004551 switch (endian) {
4552 case DEVICE_LITTLE_ENDIAN:
4553 stw_le_p(ptr, val);
4554 break;
4555 case DEVICE_BIG_ENDIAN:
4556 stw_be_p(ptr, val);
4557 break;
4558 default:
4559 stw_p(ptr, val);
4560 break;
4561 }
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004562 if (!cpu_physical_memory_is_dirty(addr1)) {
4563 /* invalidate code */
4564 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
4565 /* set dirty bit */
4566 cpu_physical_memory_set_dirty_flags(addr1,
4567 (0xff & ~CODE_DIRTY_FLAG));
4568 }
4569 }
bellardaab33092005-10-30 20:48:42 +00004570}
4571
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004572void stw_phys(target_phys_addr_t addr, uint32_t val)
4573{
4574 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
4575}
4576
4577void stw_le_phys(target_phys_addr_t addr, uint32_t val)
4578{
4579 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
4580}
4581
4582void stw_be_phys(target_phys_addr_t addr, uint32_t val)
4583{
4584 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
4585}
4586
bellardaab33092005-10-30 20:48:42 +00004587/* XXX: optimize */
Anthony Liguoric227f092009-10-01 16:12:16 -05004588void stq_phys(target_phys_addr_t addr, uint64_t val)
bellardaab33092005-10-30 20:48:42 +00004589{
4590 val = tswap64(val);
Stefan Weil71d2b722011-03-26 21:06:56 +01004591 cpu_physical_memory_write(addr, &val, 8);
bellardaab33092005-10-30 20:48:42 +00004592}
4593
Alexander Graf1e78bcc2011-07-06 09:09:23 +02004594void stq_le_phys(target_phys_addr_t addr, uint64_t val)
4595{
4596 val = cpu_to_le64(val);
4597 cpu_physical_memory_write(addr, &val, 8);
4598}
4599
4600void stq_be_phys(target_phys_addr_t addr, uint64_t val)
4601{
4602 val = cpu_to_be64(val);
4603 cpu_physical_memory_write(addr, &val, 8);
4604}
4605
aliguori5e2972f2009-03-28 17:51:36 +00004606/* virtual memory access for debug (includes writing to ROM) */
ths5fafdf22007-09-16 21:08:06 +00004607int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
bellardb448f2f2004-02-25 23:24:04 +00004608 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00004609{
4610 int l;
Anthony Liguoric227f092009-10-01 16:12:16 -05004611 target_phys_addr_t phys_addr;
j_mayer9b3c35e2007-04-07 11:21:28 +00004612 target_ulong page;
bellard13eb76e2004-01-24 15:23:36 +00004613
4614 while (len > 0) {
4615 page = addr & TARGET_PAGE_MASK;
4616 phys_addr = cpu_get_phys_page_debug(env, page);
4617 /* if no physical page mapped, return an error */
4618 if (phys_addr == -1)
4619 return -1;
4620 l = (page + TARGET_PAGE_SIZE) - addr;
4621 if (l > len)
4622 l = len;
aliguori5e2972f2009-03-28 17:51:36 +00004623 phys_addr += (addr & ~TARGET_PAGE_MASK);
aliguori5e2972f2009-03-28 17:51:36 +00004624 if (is_write)
4625 cpu_physical_memory_write_rom(phys_addr, buf, l);
4626 else
aliguori5e2972f2009-03-28 17:51:36 +00004627 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
bellard13eb76e2004-01-24 15:23:36 +00004628 len -= l;
4629 buf += l;
4630 addr += l;
4631 }
4632 return 0;
4633}
Paul Brooka68fe892010-03-01 00:08:59 +00004634#endif
bellard13eb76e2004-01-24 15:23:36 +00004635
pbrook2e70f6e2008-06-29 01:03:05 +00004636/* in deterministic execution mode, instructions doing device I/Os
4637 must be at the end of the TB */
4638void cpu_io_recompile(CPUState *env, void *retaddr)
4639{
4640 TranslationBlock *tb;
4641 uint32_t n, cflags;
4642 target_ulong pc, cs_base;
4643 uint64_t flags;
4644
4645 tb = tb_find_pc((unsigned long)retaddr);
4646 if (!tb) {
4647 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4648 retaddr);
4649 }
4650 n = env->icount_decr.u16.low + tb->icount;
Stefan Weil618ba8e2011-04-18 06:39:53 +00004651 cpu_restore_state(tb, env, (unsigned long)retaddr);
pbrook2e70f6e2008-06-29 01:03:05 +00004652 /* Calculate how many instructions had been executed before the fault
thsbf20dc02008-06-30 17:22:19 +00004653 occurred. */
pbrook2e70f6e2008-06-29 01:03:05 +00004654 n = n - env->icount_decr.u16.low;
4655 /* Generate a new TB ending on the I/O insn. */
4656 n++;
4657 /* On MIPS and SH, delay slot instructions can only be restarted if
4658 they were already the first instruction in the TB. If this is not
thsbf20dc02008-06-30 17:22:19 +00004659 the first instruction in a TB then re-execute the preceding
pbrook2e70f6e2008-06-29 01:03:05 +00004660 branch. */
4661#if defined(TARGET_MIPS)
4662 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4663 env->active_tc.PC -= 4;
4664 env->icount_decr.u16.low++;
4665 env->hflags &= ~MIPS_HFLAG_BMASK;
4666 }
4667#elif defined(TARGET_SH4)
4668 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4669 && n > 1) {
4670 env->pc -= 2;
4671 env->icount_decr.u16.low++;
4672 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4673 }
4674#endif
4675 /* This should never happen. */
4676 if (n > CF_COUNT_MASK)
4677 cpu_abort(env, "TB too big during recompile");
4678
4679 cflags = n | CF_LAST_IO;
4680 pc = tb->pc;
4681 cs_base = tb->cs_base;
4682 flags = tb->flags;
4683 tb_phys_invalidate(tb, -1);
4684 /* FIXME: In theory this could raise an exception. In practice
4685 we have already translated the block once so it's probably ok. */
4686 tb_gen_code(env, pc, cs_base, flags, cflags);
thsbf20dc02008-06-30 17:22:19 +00004687 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
pbrook2e70f6e2008-06-29 01:03:05 +00004688 the first in the TB) then we end up generating a whole new TB and
4689 repeating the fault, which is horribly inefficient.
4690 Better would be to execute just this insn uncached, or generate a
4691 second new TB. */
4692 cpu_resume_from_signal(env, NULL);
4693}
4694
Paul Brookb3755a92010-03-12 16:54:58 +00004695#if !defined(CONFIG_USER_ONLY)
4696
Stefan Weil055403b2010-10-22 23:03:32 +02004697void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
bellarde3db7222005-01-26 22:00:47 +00004698{
4699 int i, target_code_size, max_target_code_size;
4700 int direct_jmp_count, direct_jmp2_count, cross_page;
4701 TranslationBlock *tb;
ths3b46e622007-09-17 08:09:54 +00004702
bellarde3db7222005-01-26 22:00:47 +00004703 target_code_size = 0;
4704 max_target_code_size = 0;
4705 cross_page = 0;
4706 direct_jmp_count = 0;
4707 direct_jmp2_count = 0;
4708 for(i = 0; i < nb_tbs; i++) {
4709 tb = &tbs[i];
4710 target_code_size += tb->size;
4711 if (tb->size > max_target_code_size)
4712 max_target_code_size = tb->size;
4713 if (tb->page_addr[1] != -1)
4714 cross_page++;
4715 if (tb->tb_next_offset[0] != 0xffff) {
4716 direct_jmp_count++;
4717 if (tb->tb_next_offset[1] != 0xffff) {
4718 direct_jmp2_count++;
4719 }
4720 }
4721 }
4722 /* XXX: avoid using doubles ? */
bellard57fec1f2008-02-01 10:50:11 +00004723 cpu_fprintf(f, "Translation buffer state:\n");
Stefan Weil055403b2010-10-22 23:03:32 +02004724 cpu_fprintf(f, "gen code size %td/%ld\n",
bellard26a5f132008-05-28 12:30:31 +00004725 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4726 cpu_fprintf(f, "TB count %d/%d\n",
4727 nb_tbs, code_gen_max_blocks);
ths5fafdf22007-09-16 21:08:06 +00004728 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
bellarde3db7222005-01-26 22:00:47 +00004729 nb_tbs ? target_code_size / nb_tbs : 0,
4730 max_target_code_size);
Stefan Weil055403b2010-10-22 23:03:32 +02004731 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
bellarde3db7222005-01-26 22:00:47 +00004732 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4733 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
ths5fafdf22007-09-16 21:08:06 +00004734 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4735 cross_page,
bellarde3db7222005-01-26 22:00:47 +00004736 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4737 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
ths5fafdf22007-09-16 21:08:06 +00004738 direct_jmp_count,
bellarde3db7222005-01-26 22:00:47 +00004739 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4740 direct_jmp2_count,
4741 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
bellard57fec1f2008-02-01 10:50:11 +00004742 cpu_fprintf(f, "\nStatistics:\n");
bellarde3db7222005-01-26 22:00:47 +00004743 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4744 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4745 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
bellardb67d9a52008-05-23 09:57:34 +00004746 tcg_dump_info(f, cpu_fprintf);
bellarde3db7222005-01-26 22:00:47 +00004747}
4748
bellard61382a52003-10-27 21:22:23 +00004749#define MMUSUFFIX _cmmu
4750#define GETPC() NULL
4751#define env cpu_single_env
bellardb769d8f2004-10-03 15:07:13 +00004752#define SOFTMMU_CODE_ACCESS
bellard61382a52003-10-27 21:22:23 +00004753
4754#define SHIFT 0
4755#include "softmmu_template.h"
4756
4757#define SHIFT 1
4758#include "softmmu_template.h"
4759
4760#define SHIFT 2
4761#include "softmmu_template.h"
4762
4763#define SHIFT 3
4764#include "softmmu_template.h"
4765
4766#undef env
4767
4768#endif