blob: 86dd0a11c479fae169df0ceb1070824fc6e834e9 [file] [log] [blame]
aliguori05330442008-11-05 16:29:27 +00001/*
2 * QEMU KVM support
3 *
4 * Copyright IBM, Corp. 2008
aliguori5832d1f2008-11-24 19:36:26 +00005 * Red Hat, Inc. 2008
aliguori05330442008-11-05 16:29:27 +00006 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
aliguori5832d1f2008-11-24 19:36:26 +00009 * Glauber Costa <gcosta@redhat.com>
aliguori05330442008-11-05 16:29:27 +000010 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16#include <sys/types.h>
17#include <sys/ioctl.h>
18#include <sys/mman.h>
aliguori984b5182008-11-13 19:21:00 +000019#include <stdarg.h>
aliguori05330442008-11-05 16:29:27 +000020
21#include <linux/kvm.h>
22
23#include "qemu-common.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010024#include "qemu/atomic.h"
25#include "qemu/option.h"
26#include "qemu/config-file.h"
Jan Kiszkad33a1812009-05-02 00:29:37 +020027#include "hw/hw.h"
Michael S. Tsirkina2cb15b2012-12-12 14:24:50 +020028#include "hw/pci/msi.h"
Cornelia Huckd426d9f2013-07-15 17:45:03 +020029#include "hw/s390x/adapter.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010030#include "exec/gdbstub.h"
Paolo Bonzini8571ed32015-06-18 18:28:45 +020031#include "sysemu/kvm_int.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010032#include "qemu/bswap.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010033#include "exec/memory.h"
Juan Quintela747afd52013-11-04 12:59:02 +010034#include "exec/ram_addr.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010035#include "exec/address-spaces.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010036#include "qemu/event_notifier.h"
Kazuya Saito9c775722013-03-29 13:27:05 +090037#include "trace.h"
aliguori05330442008-11-05 16:29:27 +000038
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +053039#include "hw/boards.h"
40
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +020041/* This check must be after config-host.h is included */
42#ifdef CONFIG_EVENTFD
43#include <sys/eventfd.h>
44#endif
45
Stefan Weil93148aa2012-02-26 18:46:12 +010046/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
aliguorif65ed4c2008-12-09 20:09:57 +000047#define PAGE_SIZE TARGET_PAGE_SIZE
48
aliguori05330442008-11-05 16:29:27 +000049//#define DEBUG_KVM
50
51#ifdef DEBUG_KVM
Blue Swirl8c0d5772010-04-18 14:22:14 +000052#define DPRINTF(fmt, ...) \
aliguori05330442008-11-05 16:29:27 +000053 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
54#else
Blue Swirl8c0d5772010-04-18 14:22:14 +000055#define DPRINTF(fmt, ...) \
aliguori05330442008-11-05 16:29:27 +000056 do { } while (0)
57#endif
58
Jan Kiszka04fa27f2012-05-16 15:41:10 -030059#define KVM_MSI_HASHTAB_SIZE 256
60
Paolo Bonzini9d1c35d2014-10-10 12:23:35 +020061struct KVMState
aliguori05330442008-11-05 16:29:27 +000062{
Eduardo Habkostfc020862014-09-26 17:45:32 -030063 AccelState parent_obj;
64
Alex Williamsonfb541ca2013-11-22 12:12:44 -070065 int nr_slots;
aliguori05330442008-11-05 16:29:27 +000066 int fd;
67 int vmfd;
aliguorif65ed4c2008-12-09 20:09:57 +000068 int coalesced_mmio;
Sheng Yang62a27442010-01-26 19:21:16 +080069 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
Avi Kivity1cae88b2011-10-18 19:43:12 +020070 bool coalesced_flush_in_progress;
Jan Kiszkae69917e2009-05-01 20:42:15 +020071 int broken_set_mem_region;
Jan Kiszkaa0fb0022009-11-25 00:33:03 +010072 int vcpu_events;
Jan Kiszkab0b1d692010-03-01 19:10:29 +010073 int robust_singlestep;
Jan Kiszkaff44f1a2010-03-12 15:20:49 +010074 int debugregs;
aliguorie22a25c2009-03-12 20:12:48 +000075#ifdef KVM_CAP_SET_GUEST_DEBUG
76 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
77#endif
Jan Kiszka8a7c7392012-03-02 20:28:48 +010078 int pit_state2;
Sheng Yangf1665b22010-06-17 17:53:07 +080079 int xsave, xcrs;
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +020080 int many_ioeventfds;
Jan Kiszka3ab73842012-08-27 08:28:39 +020081 int intx_set_mask;
David Gibson92e4b512012-03-07 14:41:09 +000082 /* The man page (and posix) say ioctl numbers are signed int, but
83 * they're not. Linux, glibc and *BSD all treat ioctl numbers as
84 * unsigned, and treating them as signed here can break things */
Jan Kiszkae333cd62012-08-24 13:34:47 +020085 unsigned irq_set_ioctl;
James Hoganaed6efb2014-06-17 23:10:31 +010086 unsigned int sigmask_len;
Jan Kiszka84b058d2011-10-15 11:49:47 +020087#ifdef KVM_CAP_IRQ_ROUTING
88 struct kvm_irq_routing *irq_routes;
89 int nr_allocated_irq_routes;
90 uint32_t *used_gsi_bitmap;
Jan Kiszka4e2e4e62012-05-16 15:41:08 -030091 unsigned int gsi_count;
Jan Kiszka04fa27f2012-05-16 15:41:10 -030092 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
Jan Kiszka4a3adeb2012-05-16 15:41:14 -030093 bool direct_msi;
Jan Kiszka84b058d2011-10-15 11:49:47 +020094#endif
Paolo Bonzini7bbda042015-06-18 18:30:13 +020095 KVMMemoryListener memory_listener;
Paolo Bonzini9d1c35d2014-10-10 12:23:35 +020096};
aliguori05330442008-11-05 16:29:27 +000097
Jan Kiszka6a7af8c2011-02-07 12:19:25 +010098KVMState *kvm_state;
Jan Kiszka3d4b2642012-01-31 19:17:52 +010099bool kvm_kernel_irqchip;
Peter Maydell7ae26bd2012-07-26 15:35:11 +0100100bool kvm_async_interrupts_allowed;
Alexander Graf215e79c2013-04-24 22:24:12 +0200101bool kvm_halt_in_kernel_allowed;
Nikolay Nikolaev69e03ae2014-05-27 15:03:35 +0300102bool kvm_eventfds_allowed;
Peter Maydellcc7e0dd2012-07-26 15:35:14 +0100103bool kvm_irqfds_allowed;
Eric Augerf41389a2014-10-31 13:38:18 +0000104bool kvm_resamplefds_allowed;
Peter Maydell614e41b2012-07-26 15:35:15 +0100105bool kvm_msi_via_irqfd_allowed;
Peter Maydellf3e1bed2012-07-26 15:35:16 +0100106bool kvm_gsi_routing_allowed;
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +1000107bool kvm_gsi_direct_mapping;
Igor Mammedov13eed942013-04-23 10:29:36 +0200108bool kvm_allowed;
Jordan Justendf9c8b72013-05-29 01:27:25 -0700109bool kvm_readonly_mem_allowed;
Dominik Dingeld0a073a2015-03-12 13:53:49 +0100110bool kvm_vm_attributes_allowed;
aliguori05330442008-11-05 16:29:27 +0000111
Jan Kiszka94a8d392011-01-21 21:48:17 +0100112static const KVMCapabilityInfo kvm_required_capabilites[] = {
113 KVM_CAP_INFO(USER_MEMORY),
114 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
115 KVM_CAP_LAST_INFO
116};
117
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200118static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
aliguori05330442008-11-05 16:29:27 +0000119{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200120 KVMState *s = kvm_state;
aliguori05330442008-11-05 16:29:27 +0000121 int i;
122
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700123 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200124 if (kml->slots[i].memory_size == 0) {
125 return &kml->slots[i];
Jan Kiszkaa426e122011-01-04 09:32:13 +0100126 }
aliguori05330442008-11-05 16:29:27 +0000127 }
128
Igor Mammedovb8865592014-10-31 16:38:32 +0000129 return NULL;
130}
131
132bool kvm_has_free_slot(MachineState *ms)
133{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200134 KVMState *s = KVM_STATE(ms->accelerator);
135
136 return kvm_get_free_slot(&s->memory_listener);
Igor Mammedovb8865592014-10-31 16:38:32 +0000137}
138
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200139static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
Igor Mammedovb8865592014-10-31 16:38:32 +0000140{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200141 KVMSlot *slot = kvm_get_free_slot(kml);
Igor Mammedovb8865592014-10-31 16:38:32 +0000142
143 if (slot) {
144 return slot;
145 }
146
aliguorid3f8d372009-04-17 14:26:29 +0000147 fprintf(stderr, "%s: no free slot available\n", __func__);
148 abort();
149}
150
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200151static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
Avi Kivitya8170e52012-10-23 12:30:10 +0200152 hwaddr start_addr,
153 hwaddr end_addr)
aliguorid3f8d372009-04-17 14:26:29 +0000154{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200155 KVMState *s = kvm_state;
aliguorid3f8d372009-04-17 14:26:29 +0000156 int i;
157
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700158 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200159 KVMSlot *mem = &kml->slots[i];
aliguorid3f8d372009-04-17 14:26:29 +0000160
161 if (start_addr == mem->start_addr &&
162 end_addr == mem->start_addr + mem->memory_size) {
163 return mem;
164 }
165 }
166
aliguori05330442008-11-05 16:29:27 +0000167 return NULL;
168}
169
aliguori6152e2a2009-04-17 14:26:33 +0000170/*
171 * Find overlapping slot with lowest start address
172 */
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200173static KVMSlot *kvm_lookup_overlapping_slot(KVMMemoryListener *kml,
Avi Kivitya8170e52012-10-23 12:30:10 +0200174 hwaddr start_addr,
175 hwaddr end_addr)
aliguori05330442008-11-05 16:29:27 +0000176{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200177 KVMState *s = kvm_state;
aliguori6152e2a2009-04-17 14:26:33 +0000178 KVMSlot *found = NULL;
aliguori05330442008-11-05 16:29:27 +0000179 int i;
180
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700181 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200182 KVMSlot *mem = &kml->slots[i];
aliguori05330442008-11-05 16:29:27 +0000183
aliguori6152e2a2009-04-17 14:26:33 +0000184 if (mem->memory_size == 0 ||
185 (found && found->start_addr < mem->start_addr)) {
186 continue;
187 }
188
189 if (end_addr > mem->start_addr &&
190 start_addr < mem->start_addr + mem->memory_size) {
191 found = mem;
192 }
aliguori05330442008-11-05 16:29:27 +0000193 }
194
aliguori6152e2a2009-04-17 14:26:33 +0000195 return found;
aliguori05330442008-11-05 16:29:27 +0000196}
197
Avi Kivity9f213ed2011-12-15 19:55:26 +0200198int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
Avi Kivitya8170e52012-10-23 12:30:10 +0200199 hwaddr *phys_addr)
Huang Ying983dfc32010-10-11 15:31:20 -0300200{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200201 KVMMemoryListener *kml = &s->memory_listener;
Huang Ying983dfc32010-10-11 15:31:20 -0300202 int i;
203
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700204 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200205 KVMSlot *mem = &kml->slots[i];
Huang Ying983dfc32010-10-11 15:31:20 -0300206
Avi Kivity9f213ed2011-12-15 19:55:26 +0200207 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
208 *phys_addr = mem->start_addr + (ram - mem->ram);
Huang Ying983dfc32010-10-11 15:31:20 -0300209 return 1;
210 }
211 }
212
213 return 0;
214}
215
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200216static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot)
aliguori5832d1f2008-11-24 19:36:26 +0000217{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200218 KVMState *s = kvm_state;
aliguori5832d1f2008-11-24 19:36:26 +0000219 struct kvm_userspace_memory_region mem;
220
221 mem.slot = slot->slot;
222 mem.guest_phys_addr = slot->start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200223 mem.userspace_addr = (unsigned long)slot->ram;
aliguori5832d1f2008-11-24 19:36:26 +0000224 mem.flags = slot->flags;
Xiao Guangrong651eb0f2013-05-31 16:52:18 +0800225
226 if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
Jordan Justen235e8982013-05-29 01:27:26 -0700227 /* Set the slot size to 0 before setting the slot to the desired
228 * value. This is needed based on KVM commit 75d61fbc. */
229 mem.memory_size = 0;
230 kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
231 }
232 mem.memory_size = slot->memory_size;
aliguori5832d1f2008-11-24 19:36:26 +0000233 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
234}
235
Andreas Färber504134d2012-12-17 06:38:45 +0100236int kvm_init_vcpu(CPUState *cpu)
aliguori05330442008-11-05 16:29:27 +0000237{
238 KVMState *s = kvm_state;
239 long mmap_size;
240 int ret;
241
Blue Swirl8c0d5772010-04-18 14:22:14 +0000242 DPRINTF("kvm_init_vcpu\n");
aliguori05330442008-11-05 16:29:27 +0000243
Eduardo Habkostb164e482013-01-22 18:25:01 -0200244 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
aliguori05330442008-11-05 16:29:27 +0000245 if (ret < 0) {
Blue Swirl8c0d5772010-04-18 14:22:14 +0000246 DPRINTF("kvm_create_vcpu failed\n");
aliguori05330442008-11-05 16:29:27 +0000247 goto err;
248 }
249
Andreas Färber8737c512012-10-31 05:29:00 +0100250 cpu->kvm_fd = ret;
Andreas Färbera60f24b2012-12-01 05:35:08 +0100251 cpu->kvm_state = s;
Andreas Färber20d695a2012-10-31 06:57:49 +0100252 cpu->kvm_vcpu_dirty = true;
aliguori05330442008-11-05 16:29:27 +0000253
254 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
255 if (mmap_size < 0) {
Jan Kiszka748a6802011-02-01 22:15:48 +0100256 ret = mmap_size;
Blue Swirl8c0d5772010-04-18 14:22:14 +0000257 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
aliguori05330442008-11-05 16:29:27 +0000258 goto err;
259 }
260
Andreas Färberf7575c92012-12-01 06:18:14 +0100261 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
Andreas Färber8737c512012-10-31 05:29:00 +0100262 cpu->kvm_fd, 0);
Andreas Färberf7575c92012-12-01 06:18:14 +0100263 if (cpu->kvm_run == MAP_FAILED) {
aliguori05330442008-11-05 16:29:27 +0000264 ret = -errno;
Blue Swirl8c0d5772010-04-18 14:22:14 +0000265 DPRINTF("mmap'ing vcpu state failed\n");
aliguori05330442008-11-05 16:29:27 +0000266 goto err;
267 }
268
Jan Kiszkaa426e122011-01-04 09:32:13 +0100269 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
270 s->coalesced_mmio_ring =
Andreas Färberf7575c92012-12-01 06:18:14 +0100271 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100272 }
Sheng Yang62a27442010-01-26 19:21:16 +0800273
Andreas Färber20d695a2012-10-31 06:57:49 +0100274 ret = kvm_arch_init_vcpu(cpu);
aliguori05330442008-11-05 16:29:27 +0000275err:
276 return ret;
277}
278
aliguori5832d1f2008-11-24 19:36:26 +0000279/*
280 * dirty pages logging control
281 */
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300282
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200283static int kvm_mem_flags(MemoryRegion *mr)
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300284{
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200285 bool readonly = mr->readonly || memory_region_is_romd(mr);
Jordan Justen235e8982013-05-29 01:27:26 -0700286 int flags = 0;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200287
288 if (memory_region_get_dirty_log_mask(mr) != 0) {
289 flags |= KVM_MEM_LOG_DIRTY_PAGES;
290 }
Jordan Justen235e8982013-05-29 01:27:26 -0700291 if (readonly && kvm_readonly_mem_allowed) {
292 flags |= KVM_MEM_READONLY;
293 }
294 return flags;
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300295}
296
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200297static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
298 MemoryRegion *mr)
aliguori5832d1f2008-11-24 19:36:26 +0000299{
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200300 int old_flags;
301
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200302 old_flags = mem->flags;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200303 mem->flags = kvm_mem_flags(mr);
aliguori5832d1f2008-11-24 19:36:26 +0000304
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200305 /* If nothing changed effectively, no need to issue ioctl */
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200306 if (mem->flags == old_flags) {
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300307 return 0;
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200308 }
309
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200310 return kvm_set_user_memory_region(kml, mem);
aliguori5832d1f2008-11-24 19:36:26 +0000311}
312
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200313static int kvm_section_update_flags(KVMMemoryListener *kml,
314 MemoryRegionSection *section)
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300315{
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200316 hwaddr phys_addr = section->offset_within_address_space;
317 ram_addr_t size = int128_get64(section->size);
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200318 KVMSlot *mem = kvm_lookup_matching_slot(kml, phys_addr, phys_addr + size);
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300319
320 if (mem == NULL) {
Paolo Bonziniea8cb1a2015-04-27 14:51:31 +0200321 return 0;
322 } else {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200323 return kvm_slot_update_flags(kml, mem, section->mr);
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300324 }
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300325}
326
Avi Kivitya01672d2011-12-18 14:06:05 +0200327static void kvm_log_start(MemoryListener *listener,
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200328 MemoryRegionSection *section,
329 int old, int new)
aliguori5832d1f2008-11-24 19:36:26 +0000330{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200331 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
Avi Kivitya01672d2011-12-18 14:06:05 +0200332 int r;
333
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200334 if (old != 0) {
335 return;
336 }
337
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200338 r = kvm_section_update_flags(kml, section);
Avi Kivitya01672d2011-12-18 14:06:05 +0200339 if (r < 0) {
340 abort();
341 }
aliguori5832d1f2008-11-24 19:36:26 +0000342}
343
Avi Kivitya01672d2011-12-18 14:06:05 +0200344static void kvm_log_stop(MemoryListener *listener,
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200345 MemoryRegionSection *section,
346 int old, int new)
aliguori5832d1f2008-11-24 19:36:26 +0000347{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200348 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
Avi Kivitya01672d2011-12-18 14:06:05 +0200349 int r;
350
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200351 if (new != 0) {
352 return;
353 }
354
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200355 r = kvm_section_update_flags(kml, section);
Avi Kivitya01672d2011-12-18 14:06:05 +0200356 if (r < 0) {
357 abort();
358 }
aliguori5832d1f2008-11-24 19:36:26 +0000359}
360
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300361/* get kvm's dirty pages bitmap and update qemu's */
Avi Kivityffcde122011-12-19 13:18:13 +0200362static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
363 unsigned long *bitmap)
Alexander Graf96c16062009-07-27 12:49:56 +0200364{
Juan Quintelac9dd46f2013-11-05 15:45:46 +0100365 ram_addr_t start = section->offset_within_region + section->mr->ram_addr;
Juan Quintela5ff7fb72013-11-05 15:52:54 +0100366 ram_addr_t pages = int128_get64(section->size) / getpagesize();
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300367
Juan Quintela5ff7fb72013-11-05 15:52:54 +0100368 cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300369 return 0;
Alexander Graf96c16062009-07-27 12:49:56 +0200370}
371
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300372#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
373
aliguori5832d1f2008-11-24 19:36:26 +0000374/**
375 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
Blue Swirlfd4aa972011-10-16 16:04:59 +0000376 * This function updates qemu's dirty bitmap using
377 * memory_region_set_dirty(). This means all bits are set
378 * to dirty.
aliguori5832d1f2008-11-24 19:36:26 +0000379 *
aliguorid3f8d372009-04-17 14:26:29 +0000380 * @start_add: start of logged region.
aliguori5832d1f2008-11-24 19:36:26 +0000381 * @end_addr: end of logged region.
382 */
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200383static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
384 MemoryRegionSection *section)
aliguori5832d1f2008-11-24 19:36:26 +0000385{
386 KVMState *s = kvm_state;
Jan Kiszka151f7742009-05-01 20:52:47 +0200387 unsigned long size, allocated_size = 0;
Paolo Bonzini714f78c2015-06-18 18:28:44 +0200388 struct kvm_dirty_log d = {};
Jan Kiszka151f7742009-05-01 20:52:47 +0200389 KVMSlot *mem;
390 int ret = 0;
Avi Kivitya8170e52012-10-23 12:30:10 +0200391 hwaddr start_addr = section->offset_within_address_space;
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200392 hwaddr end_addr = start_addr + int128_get64(section->size);
aliguori5832d1f2008-11-24 19:36:26 +0000393
Jan Kiszka151f7742009-05-01 20:52:47 +0200394 d.dirty_bitmap = NULL;
395 while (start_addr < end_addr) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200396 mem = kvm_lookup_overlapping_slot(kml, start_addr, end_addr);
Jan Kiszka151f7742009-05-01 20:52:47 +0200397 if (mem == NULL) {
398 break;
399 }
400
Michael Tokarev51b0c602011-04-26 20:13:49 +0400401 /* XXX bad kernel interface alert
402 * For dirty bitmap, kernel allocates array of size aligned to
403 * bits-per-long. But for case when the kernel is 64bits and
404 * the userspace is 32bits, userspace can't align to the same
405 * bits-per-long, since sizeof(long) is different between kernel
406 * and user space. This way, userspace will provide buffer which
407 * may be 4 bytes less than the kernel will use, resulting in
408 * userspace memory corruption (which is not detectable by valgrind
409 * too, in most cases).
410 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
411 * a hope that sizeof(long) wont become >8 any time soon.
412 */
413 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
414 /*HOST_LONG_BITS*/ 64) / 8;
Jan Kiszka151f7742009-05-01 20:52:47 +0200415 if (!d.dirty_bitmap) {
Anthony Liguori7267c092011-08-20 22:09:37 -0500416 d.dirty_bitmap = g_malloc(size);
Jan Kiszka151f7742009-05-01 20:52:47 +0200417 } else if (size > allocated_size) {
Anthony Liguori7267c092011-08-20 22:09:37 -0500418 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
Jan Kiszka151f7742009-05-01 20:52:47 +0200419 }
420 allocated_size = size;
421 memset(d.dirty_bitmap, 0, allocated_size);
422
423 d.slot = mem->slot;
424
Michael Tokarev50212d62014-04-14 16:14:04 +0400425 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
Blue Swirl8c0d5772010-04-18 14:22:14 +0000426 DPRINTF("ioctl failed %d\n", errno);
Jan Kiszka151f7742009-05-01 20:52:47 +0200427 ret = -1;
428 break;
429 }
430
Avi Kivityffcde122011-12-19 13:18:13 +0200431 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300432 start_addr = mem->start_addr + mem->memory_size;
aliguori5832d1f2008-11-24 19:36:26 +0000433 }
Anthony Liguori7267c092011-08-20 22:09:37 -0500434 g_free(d.dirty_bitmap);
Jan Kiszka151f7742009-05-01 20:52:47 +0200435
436 return ret;
aliguori5832d1f2008-11-24 19:36:26 +0000437}
438
Avi Kivity95d29942012-10-02 18:21:54 +0200439static void kvm_coalesce_mmio_region(MemoryListener *listener,
440 MemoryRegionSection *secion,
Avi Kivitya8170e52012-10-23 12:30:10 +0200441 hwaddr start, hwaddr size)
aliguorif65ed4c2008-12-09 20:09:57 +0000442{
aliguorif65ed4c2008-12-09 20:09:57 +0000443 KVMState *s = kvm_state;
444
445 if (s->coalesced_mmio) {
446 struct kvm_coalesced_mmio_zone zone;
447
448 zone.addr = start;
449 zone.size = size;
Michael S. Tsirkin7e680752012-02-29 17:54:29 +0200450 zone.pad = 0;
aliguorif65ed4c2008-12-09 20:09:57 +0000451
Avi Kivity95d29942012-10-02 18:21:54 +0200452 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
aliguorif65ed4c2008-12-09 20:09:57 +0000453 }
aliguorif65ed4c2008-12-09 20:09:57 +0000454}
455
Avi Kivity95d29942012-10-02 18:21:54 +0200456static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
457 MemoryRegionSection *secion,
Avi Kivitya8170e52012-10-23 12:30:10 +0200458 hwaddr start, hwaddr size)
aliguorif65ed4c2008-12-09 20:09:57 +0000459{
aliguorif65ed4c2008-12-09 20:09:57 +0000460 KVMState *s = kvm_state;
461
462 if (s->coalesced_mmio) {
463 struct kvm_coalesced_mmio_zone zone;
464
465 zone.addr = start;
466 zone.size = size;
Michael S. Tsirkin7e680752012-02-29 17:54:29 +0200467 zone.pad = 0;
aliguorif65ed4c2008-12-09 20:09:57 +0000468
Avi Kivity95d29942012-10-02 18:21:54 +0200469 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
aliguorif65ed4c2008-12-09 20:09:57 +0000470 }
aliguorif65ed4c2008-12-09 20:09:57 +0000471}
472
Anthony Liguoriad7b8b32009-05-08 15:33:24 -0500473int kvm_check_extension(KVMState *s, unsigned int extension)
474{
475 int ret;
476
477 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
478 if (ret < 0) {
479 ret = 0;
480 }
481
482 return ret;
483}
484
Alexander Graf7d0a07f2014-07-14 19:15:15 +0200485int kvm_vm_check_extension(KVMState *s, unsigned int extension)
486{
487 int ret;
488
489 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
490 if (ret < 0) {
491 /* VM wide version not implemented, use global one instead */
492 ret = kvm_check_extension(s, extension);
493 }
494
495 return ret;
496}
497
Greg Kurzb680c5b2015-03-13 22:23:37 +0100498static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
499{
500#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
501 /* The kernel expects ioeventfd values in HOST_WORDS_BIGENDIAN
502 * endianness, but the memory core hands them in target endianness.
503 * For example, PPC is always treated as big-endian even if running
504 * on KVM and on PPC64LE. Correct here.
505 */
506 switch (size) {
507 case 2:
508 val = bswap16(val);
509 break;
510 case 4:
511 val = bswap32(val);
512 break;
513 }
514#endif
515 return val;
516}
517
Alexey Kardashevskiy584f2be2014-01-10 18:20:18 +1100518static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300519 bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300520{
521 int ret;
Thomas Huth03a96b82015-04-27 18:59:04 +0200522 struct kvm_ioeventfd iofd = {
523 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
524 .addr = addr,
525 .len = size,
526 .flags = 0,
527 .fd = fd,
528 };
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300529
530 if (!kvm_enabled()) {
531 return -ENOSYS;
532 }
533
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300534 if (datamatch) {
535 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
536 }
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300537 if (!assign) {
538 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
539 }
540
541 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
542
543 if (ret < 0) {
544 return -errno;
545 }
546
547 return 0;
548}
549
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300550static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300551 bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300552{
553 struct kvm_ioeventfd kick = {
Greg Kurzb680c5b2015-03-13 22:23:37 +0100554 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300555 .addr = addr,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300556 .flags = KVM_IOEVENTFD_FLAG_PIO,
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300557 .len = size,
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300558 .fd = fd,
559 };
560 int r;
561 if (!kvm_enabled()) {
562 return -ENOSYS;
563 }
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300564 if (datamatch) {
565 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
566 }
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300567 if (!assign) {
568 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
569 }
570 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
571 if (r < 0) {
572 return r;
573 }
574 return 0;
575}
576
577
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200578static int kvm_check_many_ioeventfds(void)
579{
Stefan Hajnoczid0dcac82011-01-25 16:17:14 +0000580 /* Userspace can use ioeventfd for io notification. This requires a host
581 * that supports eventfd(2) and an I/O thread; since eventfd does not
582 * support SIGIO it cannot interrupt the vcpu.
583 *
584 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200585 * can avoid creating too many ioeventfds.
586 */
Anthony Liguori12d45362011-08-22 08:24:58 -0500587#if defined(CONFIG_EVENTFD)
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200588 int ioeventfds[7];
589 int i, ret = 0;
590 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
591 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
592 if (ioeventfds[i] < 0) {
593 break;
594 }
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300595 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200596 if (ret < 0) {
597 close(ioeventfds[i]);
598 break;
599 }
600 }
601
602 /* Decide whether many devices are supported or not */
603 ret = i == ARRAY_SIZE(ioeventfds);
604
605 while (i-- > 0) {
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300606 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200607 close(ioeventfds[i]);
608 }
609 return ret;
610#else
611 return 0;
612#endif
613}
614
Jan Kiszka94a8d392011-01-21 21:48:17 +0100615static const KVMCapabilityInfo *
616kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
617{
618 while (list->name) {
619 if (!kvm_check_extension(s, list->value)) {
620 return list;
621 }
622 list++;
623 }
624 return NULL;
625}
626
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200627static void kvm_set_phys_mem(KVMMemoryListener *kml,
628 MemoryRegionSection *section, bool add)
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200629{
630 KVMState *s = kvm_state;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200631 KVMSlot *mem, old;
632 int err;
Avi Kivitya01672d2011-12-18 14:06:05 +0200633 MemoryRegion *mr = section->mr;
Jordan Justen235e8982013-05-29 01:27:26 -0700634 bool writeable = !mr->readonly && !mr->rom_device;
Avi Kivitya8170e52012-10-23 12:30:10 +0200635 hwaddr start_addr = section->offset_within_address_space;
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200636 ram_addr_t size = int128_get64(section->size);
Avi Kivity9f213ed2011-12-15 19:55:26 +0200637 void *ram = NULL;
Avi Kivity8f6f9622012-02-29 13:22:12 +0200638 unsigned delta;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200639
Gleb Natapov14542fe2010-07-28 18:13:23 +0300640 /* kvm works in page size chunks, but the function may be called
Alexander Graff2a64032014-11-07 22:12:48 +0100641 with sub-page size and unaligned start address. Pad the start
642 address to next and truncate size to previous page boundary. */
643 delta = (TARGET_PAGE_SIZE - (start_addr & ~TARGET_PAGE_MASK));
644 delta &= ~TARGET_PAGE_MASK;
Avi Kivity8f6f9622012-02-29 13:22:12 +0200645 if (delta > size) {
646 return;
647 }
648 start_addr += delta;
649 size -= delta;
650 size &= TARGET_PAGE_MASK;
651 if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
652 return;
653 }
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200654
Avi Kivitya01672d2011-12-18 14:06:05 +0200655 if (!memory_region_is_ram(mr)) {
Jordan Justen235e8982013-05-29 01:27:26 -0700656 if (writeable || !kvm_readonly_mem_allowed) {
657 return;
658 } else if (!mr->romd_mode) {
659 /* If the memory device is not in romd_mode, then we actually want
660 * to remove the kvm memory slot so all accesses will trap. */
661 add = false;
662 }
Avi Kivity9f213ed2011-12-15 19:55:26 +0200663 }
664
Avi Kivity8f6f9622012-02-29 13:22:12 +0200665 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
Avi Kivitya01672d2011-12-18 14:06:05 +0200666
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200667 while (1) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200668 mem = kvm_lookup_overlapping_slot(kml, start_addr, start_addr + size);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200669 if (!mem) {
670 break;
671 }
672
Avi Kivitya01672d2011-12-18 14:06:05 +0200673 if (add && start_addr >= mem->start_addr &&
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200674 (start_addr + size <= mem->start_addr + mem->memory_size) &&
Avi Kivity9f213ed2011-12-15 19:55:26 +0200675 (ram - start_addr == mem->ram - mem->start_addr)) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200676 /* The new slot fits into the existing one and comes with
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300677 * identical parameters - update flags and done. */
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200678 kvm_slot_update_flags(kml, mem, mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200679 return;
680 }
681
682 old = *mem;
683
Paolo Bonzini1bfbac42015-03-23 10:57:21 +0100684 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200685 kvm_physical_sync_dirty_bitmap(kml, section);
Avi Kivity3fbffb62012-01-15 16:13:59 +0200686 }
687
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200688 /* unregister the overlapping slot */
689 mem->memory_size = 0;
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200690 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200691 if (err) {
692 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
693 __func__, strerror(-err));
694 abort();
695 }
696
697 /* Workaround for older KVM versions: we can't join slots, even not by
698 * unregistering the previous ones and then registering the larger
699 * slot. We have to maintain the existing fragmentation. Sigh.
700 *
701 * This workaround assumes that the new slot starts at the same
702 * address as the first existing one. If not or if some overlapping
703 * slot comes around later, we will fail (not seen in practice so far)
704 * - and actually require a recent KVM version. */
705 if (s->broken_set_mem_region &&
Avi Kivitya01672d2011-12-18 14:06:05 +0200706 old.start_addr == start_addr && old.memory_size < size && add) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200707 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200708 mem->memory_size = old.memory_size;
709 mem->start_addr = old.start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200710 mem->ram = old.ram;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200711 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200712
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200713 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200714 if (err) {
715 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
716 strerror(-err));
717 abort();
718 }
719
720 start_addr += old.memory_size;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200721 ram += old.memory_size;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200722 size -= old.memory_size;
723 continue;
724 }
725
726 /* register prefix slot */
727 if (old.start_addr < start_addr) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200728 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200729 mem->memory_size = start_addr - old.start_addr;
730 mem->start_addr = old.start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200731 mem->ram = old.ram;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200732 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200733
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200734 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200735 if (err) {
736 fprintf(stderr, "%s: error registering prefix slot: %s\n",
737 __func__, strerror(-err));
Alexander Grafd4d68682011-04-16 10:15:11 +0200738#ifdef TARGET_PPC
739 fprintf(stderr, "%s: This is probably because your kernel's " \
740 "PAGE_SIZE is too big. Please try to use 4k " \
741 "PAGE_SIZE!\n", __func__);
742#endif
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200743 abort();
744 }
745 }
746
747 /* register suffix slot */
748 if (old.start_addr + old.memory_size > start_addr + size) {
749 ram_addr_t size_delta;
750
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200751 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200752 mem->start_addr = start_addr + size;
753 size_delta = mem->start_addr - old.start_addr;
754 mem->memory_size = old.memory_size - size_delta;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200755 mem->ram = old.ram + size_delta;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200756 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200757
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200758 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200759 if (err) {
760 fprintf(stderr, "%s: error registering suffix slot: %s\n",
761 __func__, strerror(-err));
762 abort();
763 }
764 }
765 }
766
767 /* in case the KVM bug workaround already "consumed" the new slot */
Jan Kiszkaa426e122011-01-04 09:32:13 +0100768 if (!size) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200769 return;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100770 }
Avi Kivitya01672d2011-12-18 14:06:05 +0200771 if (!add) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200772 return;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100773 }
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200774 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200775 mem->memory_size = size;
776 mem->start_addr = start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200777 mem->ram = ram;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200778 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200779
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200780 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200781 if (err) {
782 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
783 strerror(-err));
784 abort();
785 }
786}
787
Avi Kivitya01672d2011-12-18 14:06:05 +0200788static void kvm_region_add(MemoryListener *listener,
789 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200790{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200791 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
792
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200793 memory_region_ref(section->mr);
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200794 kvm_set_phys_mem(kml, section, true);
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200795}
796
Avi Kivitya01672d2011-12-18 14:06:05 +0200797static void kvm_region_del(MemoryListener *listener,
798 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200799{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200800 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
801
802 kvm_set_phys_mem(kml, section, false);
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200803 memory_region_unref(section->mr);
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200804}
805
Avi Kivitya01672d2011-12-18 14:06:05 +0200806static void kvm_log_sync(MemoryListener *listener,
807 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200808{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200809 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
Avi Kivitya01672d2011-12-18 14:06:05 +0200810 int r;
811
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200812 r = kvm_physical_sync_dirty_bitmap(kml, section);
Avi Kivitya01672d2011-12-18 14:06:05 +0200813 if (r < 0) {
814 abort();
815 }
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200816}
817
Avi Kivityd22b0962012-09-30 22:21:11 +0200818static void kvm_mem_ioeventfd_add(MemoryListener *listener,
819 MemoryRegionSection *section,
820 bool match_data, uint64_t data,
821 EventNotifier *e)
822{
823 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200824 int r;
825
Michael S. Tsirkin4b8f1c82012-03-20 14:31:38 +0200826 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200827 data, true, int128_get64(section->size),
828 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200829 if (r < 0) {
Amos Kongfa4ba922013-05-22 12:57:35 +0800830 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
831 __func__, strerror(-r));
Avi Kivity80a1ea32012-02-08 16:39:06 +0200832 abort();
833 }
834}
835
Avi Kivityd22b0962012-09-30 22:21:11 +0200836static void kvm_mem_ioeventfd_del(MemoryListener *listener,
837 MemoryRegionSection *section,
838 bool match_data, uint64_t data,
839 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200840{
Avi Kivityd22b0962012-09-30 22:21:11 +0200841 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200842 int r;
843
Michael S. Tsirkin4b8f1c82012-03-20 14:31:38 +0200844 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200845 data, false, int128_get64(section->size),
846 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200847 if (r < 0) {
848 abort();
849 }
850}
851
Avi Kivityd22b0962012-09-30 22:21:11 +0200852static void kvm_io_ioeventfd_add(MemoryListener *listener,
853 MemoryRegionSection *section,
854 bool match_data, uint64_t data,
855 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200856{
Avi Kivityd22b0962012-09-30 22:21:11 +0200857 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200858 int r;
859
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300860 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200861 data, true, int128_get64(section->size),
862 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200863 if (r < 0) {
Amos Kongfa4ba922013-05-22 12:57:35 +0800864 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
865 __func__, strerror(-r));
Avi Kivity80a1ea32012-02-08 16:39:06 +0200866 abort();
867 }
868}
869
Avi Kivityd22b0962012-09-30 22:21:11 +0200870static void kvm_io_ioeventfd_del(MemoryListener *listener,
871 MemoryRegionSection *section,
872 bool match_data, uint64_t data,
873 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200874
875{
Avi Kivityd22b0962012-09-30 22:21:11 +0200876 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200877 int r;
878
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300879 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200880 data, false, int128_get64(section->size),
881 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200882 if (r < 0) {
883 abort();
884 }
885}
886
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200887static void kvm_memory_listener_register(KVMState *s,
888 KVMMemoryListener *kml,
889 AddressSpace *as)
890{
891 int i;
892
893 kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
894
895 for (i = 0; i < s->nr_slots; i++) {
896 kml->slots[i].slot = i;
897 }
898
899 kml->listener.region_add = kvm_region_add;
900 kml->listener.region_del = kvm_region_del;
901 kml->listener.log_start = kvm_log_start;
902 kml->listener.log_stop = kvm_log_stop;
903 kml->listener.log_sync = kvm_log_sync;
904 kml->listener.priority = 10;
905
906 memory_listener_register(&kml->listener, as);
907}
Avi Kivityd22b0962012-09-30 22:21:11 +0200908
909static MemoryListener kvm_io_listener = {
Avi Kivityd22b0962012-09-30 22:21:11 +0200910 .eventfd_add = kvm_io_ioeventfd_add,
911 .eventfd_del = kvm_io_ioeventfd_del,
Avi Kivity72e22d22012-02-08 15:05:50 +0200912 .priority = 10,
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200913};
914
Andreas Färberc3affe52013-01-18 15:03:43 +0100915static void kvm_handle_interrupt(CPUState *cpu, int mask)
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200916{
Andreas Färber259186a2013-01-17 18:51:17 +0100917 cpu->interrupt_request |= mask;
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200918
Andreas Färber60e82572012-05-02 22:23:49 +0200919 if (!qemu_cpu_is_self(cpu)) {
Andreas Färberc08d7422012-05-03 04:34:15 +0200920 qemu_cpu_kick(cpu);
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200921 }
922}
923
Peter Maydell3889c3f2012-07-26 15:35:12 +0100924int kvm_set_irq(KVMState *s, int irq, int level)
Jan Kiszka84b058d2011-10-15 11:49:47 +0200925{
926 struct kvm_irq_level event;
927 int ret;
928
Peter Maydell7ae26bd2012-07-26 15:35:11 +0100929 assert(kvm_async_interrupts_enabled());
Jan Kiszka84b058d2011-10-15 11:49:47 +0200930
931 event.level = level;
932 event.irq = irq;
Jan Kiszkae333cd62012-08-24 13:34:47 +0200933 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
Jan Kiszka84b058d2011-10-15 11:49:47 +0200934 if (ret < 0) {
Peter Maydell3889c3f2012-07-26 15:35:12 +0100935 perror("kvm_set_irq");
Jan Kiszka84b058d2011-10-15 11:49:47 +0200936 abort();
937 }
938
Jan Kiszkae333cd62012-08-24 13:34:47 +0200939 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200940}
941
942#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszkad3d3bef2012-06-05 21:03:57 +0200943typedef struct KVMMSIRoute {
944 struct kvm_irq_routing_entry kroute;
945 QTAILQ_ENTRY(KVMMSIRoute) entry;
946} KVMMSIRoute;
947
Jan Kiszka84b058d2011-10-15 11:49:47 +0200948static void set_gsi(KVMState *s, unsigned int gsi)
949{
Jan Kiszka84b058d2011-10-15 11:49:47 +0200950 s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
951}
952
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300953static void clear_gsi(KVMState *s, unsigned int gsi)
954{
955 s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
956}
957
Alexander Graf7b774592013-04-16 15:58:13 +0200958void kvm_init_irq_routing(KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +0200959{
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300960 int gsi_count, i;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200961
Alexander Graf00008412014-06-06 14:46:05 +0200962 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200963 if (gsi_count > 0) {
964 unsigned int gsi_bits, i;
965
966 /* Round up so we can search ints using ffs */
Jason Baronbc8c6782012-03-28 14:18:05 -0400967 gsi_bits = ALIGN(gsi_count, 32);
Jan Kiszka84b058d2011-10-15 11:49:47 +0200968 s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
Jan Kiszka4e2e4e62012-05-16 15:41:08 -0300969 s->gsi_count = gsi_count;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200970
971 /* Mark any over-allocated bits as already in use */
972 for (i = gsi_count; i < gsi_bits; i++) {
973 set_gsi(s, i);
974 }
975 }
976
977 s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
978 s->nr_allocated_irq_routes = 0;
979
Jan Kiszka4a3adeb2012-05-16 15:41:14 -0300980 if (!s->direct_msi) {
981 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
982 QTAILQ_INIT(&s->msi_hashtab[i]);
983 }
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300984 }
985
Jan Kiszka84b058d2011-10-15 11:49:47 +0200986 kvm_arch_init_irq_routing(s);
987}
988
Alexander Grafcb925cf2013-04-17 01:11:55 +0200989void kvm_irqchip_commit_routes(KVMState *s)
Jan Kiszkae7b20302012-05-17 10:32:35 -0300990{
991 int ret;
992
993 s->irq_routes->flags = 0;
994 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
995 assert(ret == 0);
996}
997
Jan Kiszka84b058d2011-10-15 11:49:47 +0200998static void kvm_add_routing_entry(KVMState *s,
999 struct kvm_irq_routing_entry *entry)
1000{
1001 struct kvm_irq_routing_entry *new;
1002 int n, size;
1003
1004 if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
1005 n = s->nr_allocated_irq_routes * 2;
1006 if (n < 64) {
1007 n = 64;
1008 }
1009 size = sizeof(struct kvm_irq_routing);
1010 size += n * sizeof(*new);
1011 s->irq_routes = g_realloc(s->irq_routes, size);
1012 s->nr_allocated_irq_routes = n;
1013 }
1014 n = s->irq_routes->nr++;
1015 new = &s->irq_routes->entries[n];
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001016
1017 *new = *entry;
Jan Kiszka84b058d2011-10-15 11:49:47 +02001018
1019 set_gsi(s, entry->gsi);
1020}
1021
Jan Kiszkacc574072012-08-27 08:28:38 +02001022static int kvm_update_routing_entry(KVMState *s,
1023 struct kvm_irq_routing_entry *new_entry)
1024{
1025 struct kvm_irq_routing_entry *entry;
1026 int n;
1027
1028 for (n = 0; n < s->irq_routes->nr; n++) {
1029 entry = &s->irq_routes->entries[n];
1030 if (entry->gsi != new_entry->gsi) {
1031 continue;
1032 }
1033
Michael S. Tsirkin40509f72013-06-04 14:52:35 +03001034 if(!memcmp(entry, new_entry, sizeof *entry)) {
1035 return 0;
1036 }
1037
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001038 *entry = *new_entry;
Jan Kiszkacc574072012-08-27 08:28:38 +02001039
1040 kvm_irqchip_commit_routes(s);
1041
1042 return 0;
1043 }
1044
1045 return -ESRCH;
1046}
1047
Jan Kiszka1df186d2012-05-17 10:32:32 -03001048void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001049{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001050 struct kvm_irq_routing_entry e = {};
Jan Kiszka84b058d2011-10-15 11:49:47 +02001051
Jan Kiszka4e2e4e62012-05-16 15:41:08 -03001052 assert(pin < s->gsi_count);
1053
Jan Kiszka84b058d2011-10-15 11:49:47 +02001054 e.gsi = irq;
1055 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1056 e.flags = 0;
1057 e.u.irqchip.irqchip = irqchip;
1058 e.u.irqchip.pin = pin;
1059 kvm_add_routing_entry(s, &e);
1060}
1061
Jan Kiszka1e2aa8b2012-05-17 10:32:34 -03001062void kvm_irqchip_release_virq(KVMState *s, int virq)
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001063{
1064 struct kvm_irq_routing_entry *e;
1065 int i;
1066
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001067 if (kvm_gsi_direct_mapping()) {
1068 return;
1069 }
1070
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001071 for (i = 0; i < s->irq_routes->nr; i++) {
1072 e = &s->irq_routes->entries[i];
1073 if (e->gsi == virq) {
1074 s->irq_routes->nr--;
1075 *e = s->irq_routes->entries[s->irq_routes->nr];
1076 }
1077 }
1078 clear_gsi(s, virq);
1079}
1080
1081static unsigned int kvm_hash_msi(uint32_t data)
1082{
1083 /* This is optimized for IA32 MSI layout. However, no other arch shall
1084 * repeat the mistake of not providing a direct MSI injection API. */
1085 return data & 0xff;
1086}
1087
1088static void kvm_flush_dynamic_msi_routes(KVMState *s)
1089{
1090 KVMMSIRoute *route, *next;
1091 unsigned int hash;
1092
1093 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1094 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1095 kvm_irqchip_release_virq(s, route->kroute.gsi);
1096 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1097 g_free(route);
1098 }
1099 }
1100}
1101
1102static int kvm_irqchip_get_virq(KVMState *s)
1103{
1104 uint32_t *word = s->used_gsi_bitmap;
1105 int max_words = ALIGN(s->gsi_count, 32) / 32;
Stefan Hajnoczibd2a8882015-03-23 15:29:27 +00001106 int i, zeroes;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001107
马文霜bdf02632015-07-01 15:41:41 +02001108 /*
1109 * PIC and IOAPIC share the first 16 GSI numbers, thus the available
1110 * GSI numbers are more than the number of IRQ route. Allocating a GSI
1111 * number can succeed even though a new route entry cannot be added.
1112 * When this happens, flush dynamic MSI entries to free IRQ route entries.
1113 */
1114 if (!s->direct_msi && s->irq_routes->nr == s->gsi_count) {
1115 kvm_flush_dynamic_msi_routes(s);
1116 }
1117
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001118 /* Return the lowest unused GSI in the bitmap */
1119 for (i = 0; i < max_words; i++) {
Stefan Hajnoczibd2a8882015-03-23 15:29:27 +00001120 zeroes = ctz32(~word[i]);
1121 if (zeroes == 32) {
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001122 continue;
1123 }
1124
Stefan Hajnoczibd2a8882015-03-23 15:29:27 +00001125 return zeroes + i * 32;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001126 }
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001127 return -ENOSPC;
1128
1129}
1130
1131static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1132{
1133 unsigned int hash = kvm_hash_msi(msg.data);
1134 KVMMSIRoute *route;
1135
1136 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1137 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1138 route->kroute.u.msi.address_hi == (msg.address >> 32) &&
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001139 route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001140 return route;
1141 }
1142 }
1143 return NULL;
1144}
1145
1146int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1147{
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001148 struct kvm_msi msi;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001149 KVMMSIRoute *route;
1150
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001151 if (s->direct_msi) {
1152 msi.address_lo = (uint32_t)msg.address;
1153 msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001154 msi.data = le32_to_cpu(msg.data);
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001155 msi.flags = 0;
1156 memset(msi.pad, 0, sizeof(msi.pad));
1157
1158 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1159 }
1160
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001161 route = kvm_lookup_msi_route(s, msg);
1162 if (!route) {
Jan Kiszkae7b20302012-05-17 10:32:35 -03001163 int virq;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001164
1165 virq = kvm_irqchip_get_virq(s);
1166 if (virq < 0) {
1167 return virq;
1168 }
1169
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001170 route = g_malloc0(sizeof(KVMMSIRoute));
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001171 route->kroute.gsi = virq;
1172 route->kroute.type = KVM_IRQ_ROUTING_MSI;
1173 route->kroute.flags = 0;
1174 route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1175 route->kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001176 route->kroute.u.msi.data = le32_to_cpu(msg.data);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001177
1178 kvm_add_routing_entry(s, &route->kroute);
Alexander Grafcb925cf2013-04-17 01:11:55 +02001179 kvm_irqchip_commit_routes(s);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001180
1181 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1182 entry);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001183 }
1184
1185 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1186
Peter Maydell3889c3f2012-07-26 15:35:12 +01001187 return kvm_set_irq(s, route->kroute.gsi, 1);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001188}
1189
Jan Kiszka92b4e482012-05-17 10:32:33 -03001190int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1191{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001192 struct kvm_irq_routing_entry kroute = {};
Jan Kiszka92b4e482012-05-17 10:32:33 -03001193 int virq;
1194
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001195 if (kvm_gsi_direct_mapping()) {
Eric Auger1850b6b2015-06-02 14:56:23 +01001196 return kvm_arch_msi_data_to_gsi(msg.data);
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001197 }
1198
Peter Maydellf3e1bed2012-07-26 15:35:16 +01001199 if (!kvm_gsi_routing_enabled()) {
Jan Kiszka92b4e482012-05-17 10:32:33 -03001200 return -ENOSYS;
1201 }
1202
1203 virq = kvm_irqchip_get_virq(s);
1204 if (virq < 0) {
1205 return virq;
1206 }
1207
1208 kroute.gsi = virq;
1209 kroute.type = KVM_IRQ_ROUTING_MSI;
1210 kroute.flags = 0;
1211 kroute.u.msi.address_lo = (uint32_t)msg.address;
1212 kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001213 kroute.u.msi.data = le32_to_cpu(msg.data);
Frank Blaschka9e03a042015-01-09 09:04:40 +01001214 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
1215 kvm_irqchip_release_virq(s, virq);
1216 return -EINVAL;
1217 }
Jan Kiszka92b4e482012-05-17 10:32:33 -03001218
1219 kvm_add_routing_entry(s, &kroute);
Alexander Grafcb925cf2013-04-17 01:11:55 +02001220 kvm_irqchip_commit_routes(s);
Jan Kiszka92b4e482012-05-17 10:32:33 -03001221
1222 return virq;
1223}
1224
Jan Kiszkacc574072012-08-27 08:28:38 +02001225int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1226{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001227 struct kvm_irq_routing_entry kroute = {};
Jan Kiszkacc574072012-08-27 08:28:38 +02001228
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001229 if (kvm_gsi_direct_mapping()) {
1230 return 0;
1231 }
1232
Jan Kiszkacc574072012-08-27 08:28:38 +02001233 if (!kvm_irqchip_in_kernel()) {
1234 return -ENOSYS;
1235 }
1236
1237 kroute.gsi = virq;
1238 kroute.type = KVM_IRQ_ROUTING_MSI;
1239 kroute.flags = 0;
1240 kroute.u.msi.address_lo = (uint32_t)msg.address;
1241 kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001242 kroute.u.msi.data = le32_to_cpu(msg.data);
Frank Blaschka9e03a042015-01-09 09:04:40 +01001243 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
1244 return -EINVAL;
1245 }
Jan Kiszkacc574072012-08-27 08:28:38 +02001246
1247 return kvm_update_routing_entry(s, &kroute);
1248}
1249
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001250static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
1251 bool assign)
Jan Kiszka39853bb2012-05-17 10:32:36 -03001252{
1253 struct kvm_irqfd irqfd = {
1254 .fd = fd,
1255 .gsi = virq,
1256 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1257 };
1258
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001259 if (rfd != -1) {
1260 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
1261 irqfd.resamplefd = rfd;
1262 }
1263
Peter Maydellcc7e0dd2012-07-26 15:35:14 +01001264 if (!kvm_irqfds_enabled()) {
Jan Kiszka39853bb2012-05-17 10:32:36 -03001265 return -ENOSYS;
1266 }
1267
1268 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1269}
1270
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001271int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1272{
Christian Borntraegere9af2fe2014-11-20 22:10:58 +01001273 struct kvm_irq_routing_entry kroute = {};
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001274 int virq;
1275
1276 if (!kvm_gsi_routing_enabled()) {
1277 return -ENOSYS;
1278 }
1279
1280 virq = kvm_irqchip_get_virq(s);
1281 if (virq < 0) {
1282 return virq;
1283 }
1284
1285 kroute.gsi = virq;
1286 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
1287 kroute.flags = 0;
1288 kroute.u.adapter.summary_addr = adapter->summary_addr;
1289 kroute.u.adapter.ind_addr = adapter->ind_addr;
1290 kroute.u.adapter.summary_offset = adapter->summary_offset;
1291 kroute.u.adapter.ind_offset = adapter->ind_offset;
1292 kroute.u.adapter.adapter_id = adapter->adapter_id;
1293
1294 kvm_add_routing_entry(s, &kroute);
1295 kvm_irqchip_commit_routes(s);
1296
1297 return virq;
1298}
1299
Jan Kiszka84b058d2011-10-15 11:49:47 +02001300#else /* !KVM_CAP_IRQ_ROUTING */
1301
Alexander Graf7b774592013-04-16 15:58:13 +02001302void kvm_init_irq_routing(KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001303{
1304}
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001305
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001306void kvm_irqchip_release_virq(KVMState *s, int virq)
1307{
1308}
1309
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001310int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1311{
1312 abort();
1313}
Jan Kiszka92b4e482012-05-17 10:32:33 -03001314
1315int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1316{
Alex Williamsondf410672012-06-25 09:40:39 -06001317 return -ENOSYS;
Jan Kiszka92b4e482012-05-17 10:32:33 -03001318}
Jan Kiszka39853bb2012-05-17 10:32:36 -03001319
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001320int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1321{
1322 return -ENOSYS;
1323}
1324
Jan Kiszka39853bb2012-05-17 10:32:36 -03001325static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1326{
1327 abort();
1328}
Michael S. Tsirkindabe3142013-01-15 19:50:13 +02001329
1330int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1331{
1332 return -ENOSYS;
1333}
Jan Kiszka84b058d2011-10-15 11:49:47 +02001334#endif /* !KVM_CAP_IRQ_ROUTING */
1335
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001336int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
1337 EventNotifier *rn, int virq)
Jan Kiszka39853bb2012-05-17 10:32:36 -03001338{
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001339 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
1340 rn ? event_notifier_get_fd(rn) : -1, virq, true);
Jan Kiszka39853bb2012-05-17 10:32:36 -03001341}
1342
Jan Kiszkab131c742012-08-20 10:55:56 +02001343int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02001344{
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001345 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
1346 false);
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02001347}
1348
Marcel Apfelbaum446f16a2015-03-10 18:59:54 +02001349static int kvm_irqchip_create(MachineState *machine, KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001350{
Jan Kiszka84b058d2011-10-15 11:49:47 +02001351 int ret;
1352
Marcel Apfelbaum446f16a2015-03-10 18:59:54 +02001353 if (!machine_kernel_irqchip_allowed(machine) ||
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001354 (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
1355 (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
Jan Kiszka84b058d2011-10-15 11:49:47 +02001356 return 0;
1357 }
1358
Christoffer Dalld6032e02014-02-26 17:20:00 +00001359 /* First probe and see if there's a arch-specific hook to create the
1360 * in-kernel irqchip for us */
1361 ret = kvm_arch_irqchip_create(s);
Jan Kiszka84b058d2011-10-15 11:49:47 +02001362 if (ret < 0) {
Jan Kiszka84b058d2011-10-15 11:49:47 +02001363 return ret;
Christoffer Dalld6032e02014-02-26 17:20:00 +00001364 } else if (ret == 0) {
1365 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1366 if (ret < 0) {
1367 fprintf(stderr, "Create kernel irqchip failed\n");
1368 return ret;
1369 }
Jan Kiszka84b058d2011-10-15 11:49:47 +02001370 }
1371
Jan Kiszka3d4b2642012-01-31 19:17:52 +01001372 kvm_kernel_irqchip = true;
Peter Maydell7ae26bd2012-07-26 15:35:11 +01001373 /* If we have an in-kernel IRQ chip then we must have asynchronous
1374 * interrupt delivery (though the reverse is not necessarily true)
1375 */
1376 kvm_async_interrupts_allowed = true;
Alexander Graf215e79c2013-04-24 22:24:12 +02001377 kvm_halt_in_kernel_allowed = true;
Jan Kiszka84b058d2011-10-15 11:49:47 +02001378
1379 kvm_init_irq_routing(s);
1380
1381 return 0;
1382}
1383
Andrew Jones670436c2013-08-23 15:24:37 +02001384/* Find number of supported CPUs using the recommended
1385 * procedure from the kernel API documentation to cope with
1386 * older kernels that may be missing capabilities.
1387 */
1388static int kvm_recommended_vcpus(KVMState *s)
1389{
1390 int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1391 return (ret) ? ret : 4;
1392}
1393
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001394static int kvm_max_vcpus(KVMState *s)
1395{
Andrew Jones670436c2013-08-23 15:24:37 +02001396 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1397 return (ret) ? ret : kvm_recommended_vcpus(s);
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001398}
1399
Eduardo Habkostf6a1ef62014-09-26 17:45:30 -03001400static int kvm_init(MachineState *ms)
aliguori05330442008-11-05 16:29:27 +00001401{
Eduardo Habkostf6a1ef62014-09-26 17:45:30 -03001402 MachineClass *mc = MACHINE_GET_CLASS(ms);
Jan Kiszka168ccc12009-06-07 11:30:25 +02001403 static const char upgrade_note[] =
1404 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1405 "(see http://sourceforge.net/projects/kvm).\n";
Andrew Jones670436c2013-08-23 15:24:37 +02001406 struct {
1407 const char *name;
1408 int num;
1409 } num_cpus[] = {
1410 { "SMP", smp_cpus },
1411 { "hotpluggable", max_cpus },
1412 { NULL, }
1413 }, *nc = num_cpus;
1414 int soft_vcpus_limit, hard_vcpus_limit;
aliguori05330442008-11-05 16:29:27 +00001415 KVMState *s;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001416 const KVMCapabilityInfo *missing_cap;
aliguori05330442008-11-05 16:29:27 +00001417 int ret;
Paolo Bonzini7bbda042015-06-18 18:30:13 +02001418 int type = 0;
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301419 const char *kvm_type;
aliguori05330442008-11-05 16:29:27 +00001420
Eduardo Habkostfc020862014-09-26 17:45:32 -03001421 s = KVM_STATE(ms->accelerator);
aliguori05330442008-11-05 16:29:27 +00001422
David Gibson3145fcb2012-04-04 11:15:54 +10001423 /*
1424 * On systems where the kernel can support different base page
1425 * sizes, host page size may be different from TARGET_PAGE_SIZE,
1426 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
1427 * page size for the system though.
1428 */
1429 assert(TARGET_PAGE_SIZE <= getpagesize());
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -07001430 page_size_init();
David Gibson3145fcb2012-04-04 11:15:54 +10001431
James Hoganaed6efb2014-06-17 23:10:31 +01001432 s->sigmask_len = 8;
1433
aliguorie22a25c2009-03-12 20:12:48 +00001434#ifdef KVM_CAP_SET_GUEST_DEBUG
Blue Swirl72cf2d42009-09-12 07:36:22 +00001435 QTAILQ_INIT(&s->kvm_sw_breakpoints);
aliguorie22a25c2009-03-12 20:12:48 +00001436#endif
aliguori05330442008-11-05 16:29:27 +00001437 s->vmfd = -1;
Kevin Wolf40ff6d72009-12-02 12:24:42 +01001438 s->fd = qemu_open("/dev/kvm", O_RDWR);
aliguori05330442008-11-05 16:29:27 +00001439 if (s->fd == -1) {
1440 fprintf(stderr, "Could not access KVM kernel module: %m\n");
1441 ret = -errno;
1442 goto err;
1443 }
1444
1445 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1446 if (ret < KVM_API_VERSION) {
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001447 if (ret >= 0) {
aliguori05330442008-11-05 16:29:27 +00001448 ret = -EINVAL;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001449 }
aliguori05330442008-11-05 16:29:27 +00001450 fprintf(stderr, "kvm version too old\n");
1451 goto err;
1452 }
1453
1454 if (ret > KVM_API_VERSION) {
1455 ret = -EINVAL;
1456 fprintf(stderr, "kvm version not supported\n");
1457 goto err;
1458 }
1459
Alex Williamsonfb541ca2013-11-22 12:12:44 -07001460 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
1461
1462 /* If unspecified, use the default value */
1463 if (!s->nr_slots) {
1464 s->nr_slots = 32;
1465 }
1466
Andrew Jones670436c2013-08-23 15:24:37 +02001467 /* check the vcpu limits */
1468 soft_vcpus_limit = kvm_recommended_vcpus(s);
1469 hard_vcpus_limit = kvm_max_vcpus(s);
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001470
Andrew Jones670436c2013-08-23 15:24:37 +02001471 while (nc->name) {
1472 if (nc->num > soft_vcpus_limit) {
1473 fprintf(stderr,
1474 "Warning: Number of %s cpus requested (%d) exceeds "
1475 "the recommended cpus supported by KVM (%d)\n",
1476 nc->name, nc->num, soft_vcpus_limit);
1477
1478 if (nc->num > hard_vcpus_limit) {
Andrew Jones670436c2013-08-23 15:24:37 +02001479 fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
1480 "the maximum cpus supported by KVM (%d)\n",
1481 nc->name, nc->num, hard_vcpus_limit);
Marcelo Tosatti9ba3cf52014-02-25 23:22:07 -03001482 exit(1);
Andrew Jones670436c2013-08-23 15:24:37 +02001483 }
1484 }
1485 nc++;
Marcelo Tosatti7dc52522013-08-12 16:56:31 -03001486 }
1487
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301488 kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type");
Marcel Apfelbaumf1e29872014-04-09 20:34:52 +03001489 if (mc->kvm_type) {
1490 type = mc->kvm_type(kvm_type);
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301491 } else if (kvm_type) {
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001492 ret = -EINVAL;
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301493 fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
1494 goto err;
1495 }
1496
thomas knych94ccff12014-01-09 13:14:23 -08001497 do {
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301498 ret = kvm_ioctl(s, KVM_CREATE_VM, type);
thomas knych94ccff12014-01-09 13:14:23 -08001499 } while (ret == -EINTR);
1500
1501 if (ret < 0) {
Alexander Graf521f4382014-01-27 15:18:09 +01001502 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
thomas knych94ccff12014-01-09 13:14:23 -08001503 strerror(-ret));
1504
Alexander Graf0104dca2010-04-01 18:42:37 +02001505#ifdef TARGET_S390X
Cornelia Huck2c80e992015-04-23 17:03:46 +02001506 if (ret == -EINVAL) {
1507 fprintf(stderr,
1508 "Host kernel setup problem detected. Please verify:\n");
1509 fprintf(stderr, "- for kernels supporting the switch_amode or"
1510 " user_mode parameters, whether\n");
1511 fprintf(stderr,
1512 " user space is running in primary address space\n");
1513 fprintf(stderr,
1514 "- for kernels supporting the vm.allocate_pgste sysctl, "
1515 "whether it is enabled\n");
1516 }
Alexander Graf0104dca2010-04-01 18:42:37 +02001517#endif
aliguori05330442008-11-05 16:29:27 +00001518 goto err;
Alexander Graf0104dca2010-04-01 18:42:37 +02001519 }
aliguori05330442008-11-05 16:29:27 +00001520
thomas knych94ccff12014-01-09 13:14:23 -08001521 s->vmfd = ret;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001522 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1523 if (!missing_cap) {
1524 missing_cap =
1525 kvm_check_extension_list(s, kvm_arch_required_capabilities);
1526 }
1527 if (missing_cap) {
Anthony Liguoriad7b8b32009-05-08 15:33:24 -05001528 ret = -EINVAL;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001529 fprintf(stderr, "kvm does not support %s\n%s",
1530 missing_cap->name, upgrade_note);
aliguori05330442008-11-05 16:29:27 +00001531 goto err;
1532 }
1533
Anthony Liguoriad7b8b32009-05-08 15:33:24 -05001534 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
aliguorif65ed4c2008-12-09 20:09:57 +00001535
Jan Kiszkae69917e2009-05-01 20:42:15 +02001536 s->broken_set_mem_region = 1;
Lai Jiangshan14a09512010-12-10 15:52:36 +08001537 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
Jan Kiszkae69917e2009-05-01 20:42:15 +02001538 if (ret > 0) {
1539 s->broken_set_mem_region = 0;
1540 }
Jan Kiszkae69917e2009-05-01 20:42:15 +02001541
Jan Kiszkaa0fb0022009-11-25 00:33:03 +01001542#ifdef KVM_CAP_VCPU_EVENTS
1543 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1544#endif
1545
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001546 s->robust_singlestep =
1547 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001548
Jan Kiszkaff44f1a2010-03-12 15:20:49 +01001549#ifdef KVM_CAP_DEBUGREGS
1550 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1551#endif
1552
Sheng Yangf1665b22010-06-17 17:53:07 +08001553#ifdef KVM_CAP_XSAVE
1554 s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
1555#endif
1556
Sheng Yangf1665b22010-06-17 17:53:07 +08001557#ifdef KVM_CAP_XCRS
1558 s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
1559#endif
1560
Jan Kiszka8a7c7392012-03-02 20:28:48 +01001561#ifdef KVM_CAP_PIT_STATE2
1562 s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
1563#endif
1564
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001565#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001566 s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001567#endif
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001568
Jan Kiszka3ab73842012-08-27 08:28:39 +02001569 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1570
Jan Kiszkae333cd62012-08-24 13:34:47 +02001571 s->irq_set_ioctl = KVM_IRQ_LINE;
Peter Maydell8732fbd2012-08-15 12:08:13 +01001572 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
Jan Kiszkae333cd62012-08-24 13:34:47 +02001573 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
Peter Maydell8732fbd2012-08-15 12:08:13 +01001574 }
1575
Jordan Justendf9c8b72013-05-29 01:27:25 -07001576#ifdef KVM_CAP_READONLY_MEM
1577 kvm_readonly_mem_allowed =
1578 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1579#endif
1580
Nikolay Nikolaev69e03ae2014-05-27 15:03:35 +03001581 kvm_eventfds_allowed =
1582 (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
1583
Eric Augerf41389a2014-10-31 13:38:18 +00001584 kvm_irqfds_allowed =
1585 (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
1586
1587 kvm_resamplefds_allowed =
1588 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
1589
Dominik Dingeld0a073a2015-03-12 13:53:49 +01001590 kvm_vm_attributes_allowed =
1591 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
1592
Marcel Apfelbaumb16565b2015-02-04 17:43:51 +02001593 ret = kvm_arch_init(ms, s);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001594 if (ret < 0) {
aliguori05330442008-11-05 16:29:27 +00001595 goto err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001596 }
aliguori05330442008-11-05 16:29:27 +00001597
Marcel Apfelbaum446f16a2015-03-10 18:59:54 +02001598 ret = kvm_irqchip_create(ms, s);
Jan Kiszka84b058d2011-10-15 11:49:47 +02001599 if (ret < 0) {
1600 goto err;
1601 }
1602
aliguori05330442008-11-05 16:29:27 +00001603 kvm_state = s;
Paolo Bonzini7bbda042015-06-18 18:30:13 +02001604
1605 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
1606 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
1607 s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region;
1608 s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region;
1609
1610 kvm_memory_listener_register(s, &s->memory_listener,
1611 &address_space_memory);
1612 memory_listener_register(&kvm_io_listener,
1613 &address_space_io);
aliguori05330442008-11-05 16:29:27 +00001614
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +02001615 s->many_ioeventfds = kvm_check_many_ioeventfds();
1616
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +02001617 cpu_interrupt_handler = kvm_handle_interrupt;
1618
aliguori05330442008-11-05 16:29:27 +00001619 return 0;
1620
1621err:
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001622 assert(ret < 0);
Stefan Weil6d1cc322012-09-03 22:40:40 +02001623 if (s->vmfd >= 0) {
1624 close(s->vmfd);
1625 }
1626 if (s->fd != -1) {
1627 close(s->fd);
aliguori05330442008-11-05 16:29:27 +00001628 }
Paolo Bonzini7bbda042015-06-18 18:30:13 +02001629 g_free(s->memory_listener.slots);
aliguori05330442008-11-05 16:29:27 +00001630
1631 return ret;
1632}
1633
James Hoganaed6efb2014-06-17 23:10:31 +01001634void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
1635{
1636 s->sigmask_len = sigmask_len;
1637}
1638
Paolo Bonzini4c663752015-04-08 13:30:58 +02001639static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
1640 int size, uint32_t count)
aliguori05330442008-11-05 16:29:27 +00001641{
1642 int i;
1643 uint8_t *ptr = data;
1644
1645 for (i = 0; i < count; i++) {
Paolo Bonzini4c663752015-04-08 13:30:58 +02001646 address_space_rw(&address_space_io, port, attrs,
Peter Maydell5c9eb022015-04-26 16:49:24 +01001647 ptr, size,
Jan Kiszka354678c2013-08-13 14:43:57 +02001648 direction == KVM_EXIT_IO_OUT);
aliguori05330442008-11-05 16:29:27 +00001649 ptr += size;
1650 }
aliguori05330442008-11-05 16:29:27 +00001651}
1652
Andreas Färber5326ab52013-05-27 01:55:29 +02001653static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001654{
Radim Krčmář977c7b62014-01-21 18:11:31 +01001655 fprintf(stderr, "KVM internal error. Suberror: %d\n",
1656 run->internal.suberror);
1657
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001658 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1659 int i;
1660
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001661 for (i = 0; i < run->internal.ndata; ++i) {
1662 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1663 i, (uint64_t)run->internal.data[i]);
1664 }
1665 }
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001666 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1667 fprintf(stderr, "emulation failure\n");
Andreas Färber20d695a2012-10-31 06:57:49 +01001668 if (!kvm_arch_stop_on_emulation_error(cpu)) {
Andreas Färber878096e2013-05-27 01:33:50 +02001669 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001670 return EXCP_INTERRUPT;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001671 }
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001672 }
1673 /* FIXME: Should trigger a qmp message to let management know
1674 * something went wrong.
1675 */
Jan Kiszka73aaec42011-01-21 21:48:06 +01001676 return -1;
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001677}
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001678
Sheng Yang62a27442010-01-26 19:21:16 +08001679void kvm_flush_coalesced_mmio_buffer(void)
aliguorif65ed4c2008-12-09 20:09:57 +00001680{
aliguorif65ed4c2008-12-09 20:09:57 +00001681 KVMState *s = kvm_state;
Avi Kivity1cae88b2011-10-18 19:43:12 +02001682
1683 if (s->coalesced_flush_in_progress) {
1684 return;
1685 }
1686
1687 s->coalesced_flush_in_progress = true;
1688
Sheng Yang62a27442010-01-26 19:21:16 +08001689 if (s->coalesced_mmio_ring) {
1690 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
aliguorif65ed4c2008-12-09 20:09:57 +00001691 while (ring->first != ring->last) {
1692 struct kvm_coalesced_mmio *ent;
1693
1694 ent = &ring->coalesced_mmio[ring->first];
1695
1696 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
Marcelo Tosatti85199472010-02-22 13:57:54 -03001697 smp_wmb();
aliguorif65ed4c2008-12-09 20:09:57 +00001698 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1699 }
1700 }
Avi Kivity1cae88b2011-10-18 19:43:12 +02001701
1702 s->coalesced_flush_in_progress = false;
aliguorif65ed4c2008-12-09 20:09:57 +00001703}
1704
Andreas Färber20d695a2012-10-31 06:57:49 +01001705static void do_kvm_cpu_synchronize_state(void *arg)
Avi Kivity4c0960c2009-08-17 23:19:53 +03001706{
Andreas Färber20d695a2012-10-31 06:57:49 +01001707 CPUState *cpu = arg;
Jan Kiszka2705d562010-05-04 09:45:23 -03001708
Andreas Färber20d695a2012-10-31 06:57:49 +01001709 if (!cpu->kvm_vcpu_dirty) {
1710 kvm_arch_get_registers(cpu);
1711 cpu->kvm_vcpu_dirty = true;
Avi Kivity4c0960c2009-08-17 23:19:53 +03001712 }
1713}
1714
Andreas Färberdd1750d2013-05-01 13:45:44 +02001715void kvm_cpu_synchronize_state(CPUState *cpu)
Jan Kiszka2705d562010-05-04 09:45:23 -03001716{
Andreas Färber20d695a2012-10-31 06:57:49 +01001717 if (!cpu->kvm_vcpu_dirty) {
1718 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001719 }
Jan Kiszka2705d562010-05-04 09:45:23 -03001720}
1721
David Hildenbrandc8e20852014-08-20 14:55:25 +02001722static void do_kvm_cpu_synchronize_post_reset(void *arg)
1723{
1724 CPUState *cpu = arg;
1725
1726 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1727 cpu->kvm_vcpu_dirty = false;
1728}
1729
Igor Mammedov3f24a582013-04-11 16:51:41 +02001730void kvm_cpu_synchronize_post_reset(CPUState *cpu)
Jan Kiszkaea375f92010-03-01 19:10:30 +01001731{
David Hildenbrandc8e20852014-08-20 14:55:25 +02001732 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, cpu);
1733}
1734
1735static void do_kvm_cpu_synchronize_post_init(void *arg)
1736{
1737 CPUState *cpu = arg;
1738
1739 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
Andreas Färber20d695a2012-10-31 06:57:49 +01001740 cpu->kvm_vcpu_dirty = false;
Jan Kiszkaea375f92010-03-01 19:10:30 +01001741}
1742
Igor Mammedov3f24a582013-04-11 16:51:41 +02001743void kvm_cpu_synchronize_post_init(CPUState *cpu)
Jan Kiszkaea375f92010-03-01 19:10:30 +01001744{
David Hildenbrandc8e20852014-08-20 14:55:25 +02001745 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
Jan Kiszkaea375f92010-03-01 19:10:30 +01001746}
1747
Marcelo Tosattide9d61e2014-09-05 10:52:46 -03001748void kvm_cpu_clean_state(CPUState *cpu)
1749{
1750 cpu->kvm_vcpu_dirty = false;
1751}
1752
Andreas Färber1458c362013-05-26 23:46:55 +02001753int kvm_cpu_exec(CPUState *cpu)
aliguori05330442008-11-05 16:29:27 +00001754{
Andreas Färberf7575c92012-12-01 06:18:14 +01001755 struct kvm_run *run = cpu->kvm_run;
Jan Kiszka7cbb5332011-03-15 12:26:25 +01001756 int ret, run_ret;
aliguori05330442008-11-05 16:29:27 +00001757
Blue Swirl8c0d5772010-04-18 14:22:14 +00001758 DPRINTF("kvm_cpu_exec()\n");
aliguori05330442008-11-05 16:29:27 +00001759
Andreas Färber20d695a2012-10-31 06:57:49 +01001760 if (kvm_arch_process_async_events(cpu)) {
Andreas Färberfcd7d002012-12-17 08:02:44 +01001761 cpu->exit_request = 0;
Jan Kiszka6792a572011-02-07 12:19:18 +01001762 return EXCP_HLT;
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001763 }
1764
Jan Kiszka4b8523e2015-06-18 18:47:23 +02001765 qemu_mutex_unlock_iothread();
1766
aliguori05330442008-11-05 16:29:27 +00001767 do {
Paolo Bonzini4c663752015-04-08 13:30:58 +02001768 MemTxAttrs attrs;
1769
Andreas Färber20d695a2012-10-31 06:57:49 +01001770 if (cpu->kvm_vcpu_dirty) {
1771 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1772 cpu->kvm_vcpu_dirty = false;
Avi Kivity4c0960c2009-08-17 23:19:53 +03001773 }
1774
Andreas Färber20d695a2012-10-31 06:57:49 +01001775 kvm_arch_pre_run(cpu, run);
Andreas Färberfcd7d002012-12-17 08:02:44 +01001776 if (cpu->exit_request) {
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001777 DPRINTF("interrupt exit requested\n");
1778 /*
1779 * KVM requires us to reenter the kernel after IO exits to complete
1780 * instruction emulation. This self-signal will ensure that we
1781 * leave ASAP again.
1782 */
1783 qemu_cpu_kick_self();
1784 }
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001785
Andreas Färber1bc22652012-10-31 06:06:49 +01001786 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001787
Paolo Bonzini4c663752015-04-08 13:30:58 +02001788 attrs = kvm_arch_post_run(cpu, run);
aliguori05330442008-11-05 16:29:27 +00001789
Jan Kiszka7cbb5332011-03-15 12:26:25 +01001790 if (run_ret < 0) {
Jan Kiszkadc77d342011-03-15 12:26:26 +01001791 if (run_ret == -EINTR || run_ret == -EAGAIN) {
1792 DPRINTF("io window exit\n");
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001793 ret = EXCP_INTERRUPT;
Jan Kiszkadc77d342011-03-15 12:26:26 +01001794 break;
1795 }
Michael Ellerman7b011fb2011-12-16 11:20:20 +11001796 fprintf(stderr, "error: kvm run failed %s\n",
1797 strerror(-run_ret));
Laurent Vivierdae02ba2015-05-18 21:06:47 +02001798#ifdef TARGET_PPC
1799 if (run_ret == -EBUSY) {
1800 fprintf(stderr,
1801 "This is probably because your SMT is enabled.\n"
1802 "VCPU can only run on primary threads with all "
1803 "secondary threads offline.\n");
1804 }
1805#endif
Paolo Bonzinia85e1302014-08-29 15:58:20 +02001806 ret = -1;
1807 break;
aliguori05330442008-11-05 16:29:27 +00001808 }
1809
Kazuya Saitob76ac802013-03-29 13:27:52 +09001810 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
aliguori05330442008-11-05 16:29:27 +00001811 switch (run->exit_reason) {
1812 case KVM_EXIT_IO:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001813 DPRINTF("handle_io\n");
Jan Kiszka80b7d2e2015-06-18 18:47:24 +02001814 /* Called outside BQL */
Paolo Bonzini4c663752015-04-08 13:30:58 +02001815 kvm_handle_io(run->io.port, attrs,
Jan Kiszkab30e93e2011-02-01 22:16:01 +01001816 (uint8_t *)run + run->io.data_offset,
1817 run->io.direction,
1818 run->io.size,
1819 run->io.count);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001820 ret = 0;
aliguori05330442008-11-05 16:29:27 +00001821 break;
1822 case KVM_EXIT_MMIO:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001823 DPRINTF("handle_mmio\n");
Paolo Bonzinide7ea882015-06-18 18:47:26 +02001824 /* Called outside BQL */
Paolo Bonzini4c663752015-04-08 13:30:58 +02001825 address_space_rw(&address_space_memory,
1826 run->mmio.phys_addr, attrs,
1827 run->mmio.data,
1828 run->mmio.len,
1829 run->mmio.is_write);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001830 ret = 0;
aliguori05330442008-11-05 16:29:27 +00001831 break;
1832 case KVM_EXIT_IRQ_WINDOW_OPEN:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001833 DPRINTF("irq_window_open\n");
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001834 ret = EXCP_INTERRUPT;
aliguori05330442008-11-05 16:29:27 +00001835 break;
1836 case KVM_EXIT_SHUTDOWN:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001837 DPRINTF("shutdown\n");
aliguori05330442008-11-05 16:29:27 +00001838 qemu_system_reset_request();
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001839 ret = EXCP_INTERRUPT;
aliguori05330442008-11-05 16:29:27 +00001840 break;
1841 case KVM_EXIT_UNKNOWN:
Jan Kiszkabb44e0d2011-01-21 21:48:07 +01001842 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1843 (uint64_t)run->hw.hardware_exit_reason);
Jan Kiszka73aaec42011-01-21 21:48:06 +01001844 ret = -1;
aliguori05330442008-11-05 16:29:27 +00001845 break;
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001846 case KVM_EXIT_INTERNAL_ERROR:
Andreas Färber5326ab52013-05-27 01:55:29 +02001847 ret = kvm_handle_internal_error(cpu, run);
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001848 break;
Pranavkumar Sawargaonkar99040442014-06-19 18:06:25 +01001849 case KVM_EXIT_SYSTEM_EVENT:
1850 switch (run->system_event.type) {
1851 case KVM_SYSTEM_EVENT_SHUTDOWN:
1852 qemu_system_shutdown_request();
1853 ret = EXCP_INTERRUPT;
1854 break;
1855 case KVM_SYSTEM_EVENT_RESET:
1856 qemu_system_reset_request();
1857 ret = EXCP_INTERRUPT;
1858 break;
1859 default:
1860 DPRINTF("kvm_arch_handle_exit\n");
1861 ret = kvm_arch_handle_exit(cpu, run);
1862 break;
1863 }
1864 break;
aliguori05330442008-11-05 16:29:27 +00001865 default:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001866 DPRINTF("kvm_arch_handle_exit\n");
Andreas Färber20d695a2012-10-31 06:57:49 +01001867 ret = kvm_arch_handle_exit(cpu, run);
aliguori05330442008-11-05 16:29:27 +00001868 break;
1869 }
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001870 } while (ret == 0);
aliguori05330442008-11-05 16:29:27 +00001871
Jan Kiszka4b8523e2015-06-18 18:47:23 +02001872 qemu_mutex_lock_iothread();
1873
Jan Kiszka73aaec42011-01-21 21:48:06 +01001874 if (ret < 0) {
Andreas Färber878096e2013-05-27 01:33:50 +02001875 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
Luiz Capitulino0461d5a2011-09-30 14:45:27 -03001876 vm_stop(RUN_STATE_INTERNAL_ERROR);
Jan Kiszka73aaec42011-01-21 21:48:06 +01001877 }
aliguoribecfc392008-11-10 15:55:14 +00001878
Andreas Färberfcd7d002012-12-17 08:02:44 +01001879 cpu->exit_request = 0;
aliguori05330442008-11-05 16:29:27 +00001880 return ret;
1881}
1882
aliguori984b5182008-11-13 19:21:00 +00001883int kvm_ioctl(KVMState *s, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001884{
1885 int ret;
aliguori984b5182008-11-13 19:21:00 +00001886 void *arg;
1887 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001888
aliguori984b5182008-11-13 19:21:00 +00001889 va_start(ap, type);
1890 arg = va_arg(ap, void *);
1891 va_end(ap);
1892
Kazuya Saito9c775722013-03-29 13:27:05 +09001893 trace_kvm_ioctl(type, arg);
aliguori984b5182008-11-13 19:21:00 +00001894 ret = ioctl(s->fd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001895 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001896 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001897 }
aliguori05330442008-11-05 16:29:27 +00001898 return ret;
1899}
1900
aliguori984b5182008-11-13 19:21:00 +00001901int kvm_vm_ioctl(KVMState *s, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001902{
1903 int ret;
aliguori984b5182008-11-13 19:21:00 +00001904 void *arg;
1905 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001906
aliguori984b5182008-11-13 19:21:00 +00001907 va_start(ap, type);
1908 arg = va_arg(ap, void *);
1909 va_end(ap);
1910
Kazuya Saito9c775722013-03-29 13:27:05 +09001911 trace_kvm_vm_ioctl(type, arg);
aliguori984b5182008-11-13 19:21:00 +00001912 ret = ioctl(s->vmfd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001913 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001914 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001915 }
aliguori05330442008-11-05 16:29:27 +00001916 return ret;
1917}
1918
Andreas Färber1bc22652012-10-31 06:06:49 +01001919int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001920{
1921 int ret;
aliguori984b5182008-11-13 19:21:00 +00001922 void *arg;
1923 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001924
aliguori984b5182008-11-13 19:21:00 +00001925 va_start(ap, type);
1926 arg = va_arg(ap, void *);
1927 va_end(ap);
1928
Kazuya Saito9c775722013-03-29 13:27:05 +09001929 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
Andreas Färber8737c512012-10-31 05:29:00 +01001930 ret = ioctl(cpu->kvm_fd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001931 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001932 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001933 }
aliguori05330442008-11-05 16:29:27 +00001934 return ret;
1935}
aliguoribd322082008-12-04 20:33:06 +00001936
Christoffer Dall0a6a7cc2014-02-26 17:20:00 +00001937int kvm_device_ioctl(int fd, int type, ...)
1938{
1939 int ret;
1940 void *arg;
1941 va_list ap;
1942
1943 va_start(ap, type);
1944 arg = va_arg(ap, void *);
1945 va_end(ap);
1946
1947 trace_kvm_device_ioctl(fd, type, arg);
1948 ret = ioctl(fd, type, arg);
1949 if (ret == -1) {
1950 ret = -errno;
1951 }
1952 return ret;
1953}
1954
Dominik Dingeld0a073a2015-03-12 13:53:49 +01001955int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
1956{
1957 int ret;
1958 struct kvm_device_attr attribute = {
1959 .group = group,
1960 .attr = attr,
1961 };
1962
1963 if (!kvm_vm_attributes_allowed) {
1964 return 0;
1965 }
1966
1967 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
1968 /* kvm returns 0 on success for HAS_DEVICE_ATTR */
1969 return ret ? 0 : 1;
1970}
1971
aliguoribd322082008-12-04 20:33:06 +00001972int kvm_has_sync_mmu(void)
1973{
Jan Kiszka94a8d392011-01-21 21:48:17 +01001974 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
aliguoribd322082008-12-04 20:33:06 +00001975}
aliguorie22a25c2009-03-12 20:12:48 +00001976
Jan Kiszkaa0fb0022009-11-25 00:33:03 +01001977int kvm_has_vcpu_events(void)
1978{
1979 return kvm_state->vcpu_events;
1980}
1981
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001982int kvm_has_robust_singlestep(void)
1983{
1984 return kvm_state->robust_singlestep;
1985}
1986
Jan Kiszkaff44f1a2010-03-12 15:20:49 +01001987int kvm_has_debugregs(void)
1988{
1989 return kvm_state->debugregs;
1990}
1991
Sheng Yangf1665b22010-06-17 17:53:07 +08001992int kvm_has_xsave(void)
1993{
1994 return kvm_state->xsave;
1995}
1996
1997int kvm_has_xcrs(void)
1998{
1999 return kvm_state->xcrs;
2000}
2001
Jan Kiszka8a7c7392012-03-02 20:28:48 +01002002int kvm_has_pit_state2(void)
2003{
2004 return kvm_state->pit_state2;
2005}
2006
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +02002007int kvm_has_many_ioeventfds(void)
2008{
2009 if (!kvm_enabled()) {
2010 return 0;
2011 }
2012 return kvm_state->many_ioeventfds;
2013}
2014
Jan Kiszka84b058d2011-10-15 11:49:47 +02002015int kvm_has_gsi_routing(void)
2016{
Alexander Grafa9c5eb02012-01-25 18:28:05 +01002017#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszka84b058d2011-10-15 11:49:47 +02002018 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
Alexander Grafa9c5eb02012-01-25 18:28:05 +01002019#else
2020 return false;
2021#endif
Jan Kiszka84b058d2011-10-15 11:49:47 +02002022}
2023
Jan Kiszka3ab73842012-08-27 08:28:39 +02002024int kvm_has_intx_set_mask(void)
2025{
2026 return kvm_state->intx_set_mask;
2027}
2028
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002029void kvm_setup_guest_memory(void *start, size_t size)
2030{
2031 if (!kvm_has_sync_mmu()) {
Andreas Färbere78815a2010-09-25 11:26:05 +00002032 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002033
2034 if (ret) {
Andreas Färbere78815a2010-09-25 11:26:05 +00002035 perror("qemu_madvise");
2036 fprintf(stderr,
2037 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002038 exit(1);
2039 }
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002040 }
2041}
2042
aliguorie22a25c2009-03-12 20:12:48 +00002043#ifdef KVM_CAP_SET_GUEST_DEBUG
Andreas Färbera60f24b2012-12-01 05:35:08 +01002044struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
aliguorie22a25c2009-03-12 20:12:48 +00002045 target_ulong pc)
2046{
2047 struct kvm_sw_breakpoint *bp;
2048
Andreas Färbera60f24b2012-12-01 05:35:08 +01002049 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
Jan Kiszkaa426e122011-01-04 09:32:13 +01002050 if (bp->pc == pc) {
aliguorie22a25c2009-03-12 20:12:48 +00002051 return bp;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002052 }
aliguorie22a25c2009-03-12 20:12:48 +00002053 }
2054 return NULL;
2055}
2056
Andreas Färbera60f24b2012-12-01 05:35:08 +01002057int kvm_sw_breakpoints_active(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002058{
Andreas Färbera60f24b2012-12-01 05:35:08 +01002059 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
aliguorie22a25c2009-03-12 20:12:48 +00002060}
2061
Glauber Costa452e4752009-07-16 17:55:28 -04002062struct kvm_set_guest_debug_data {
2063 struct kvm_guest_debug dbg;
Andreas Färbera60f24b2012-12-01 05:35:08 +01002064 CPUState *cpu;
Glauber Costa452e4752009-07-16 17:55:28 -04002065 int err;
2066};
2067
2068static void kvm_invoke_set_guest_debug(void *data)
2069{
2070 struct kvm_set_guest_debug_data *dbg_data = data;
Jan Kiszkab3807722009-09-17 20:05:58 +02002071
Andreas Färbera60f24b2012-12-01 05:35:08 +01002072 dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
2073 &dbg_data->dbg);
Glauber Costa452e4752009-07-16 17:55:28 -04002074}
2075
Stefan Weil38e478e2013-07-25 20:50:21 +02002076int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
aliguorie22a25c2009-03-12 20:12:48 +00002077{
Glauber Costa452e4752009-07-16 17:55:28 -04002078 struct kvm_set_guest_debug_data data;
aliguorie22a25c2009-03-12 20:12:48 +00002079
Jan Kiszkab0b1d692010-03-01 19:10:29 +01002080 data.dbg.control = reinject_trap;
aliguorie22a25c2009-03-12 20:12:48 +00002081
Andreas Färbered2803d2013-06-21 20:20:45 +02002082 if (cpu->singlestep_enabled) {
Jan Kiszkab0b1d692010-03-01 19:10:29 +01002083 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2084 }
Andreas Färber20d695a2012-10-31 06:57:49 +01002085 kvm_arch_update_guest_debug(cpu, &data.dbg);
Andreas Färbera60f24b2012-12-01 05:35:08 +01002086 data.cpu = cpu;
aliguorie22a25c2009-03-12 20:12:48 +00002087
Andreas Färberf100f0b2012-05-03 14:58:47 +02002088 run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
Glauber Costa452e4752009-07-16 17:55:28 -04002089 return data.err;
aliguorie22a25c2009-03-12 20:12:48 +00002090}
2091
Andreas Färber62278812013-06-27 17:12:06 +02002092int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002093 target_ulong len, int type)
2094{
2095 struct kvm_sw_breakpoint *bp;
aliguorie22a25c2009-03-12 20:12:48 +00002096 int err;
2097
2098 if (type == GDB_BREAKPOINT_SW) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002099 bp = kvm_find_sw_breakpoint(cpu, addr);
aliguorie22a25c2009-03-12 20:12:48 +00002100 if (bp) {
2101 bp->use_count++;
2102 return 0;
2103 }
2104
Anthony Liguori7267c092011-08-20 22:09:37 -05002105 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
aliguorie22a25c2009-03-12 20:12:48 +00002106 bp->pc = addr;
2107 bp->use_count = 1;
Andreas Färber80b7cd72013-06-19 17:37:31 +02002108 err = kvm_arch_insert_sw_breakpoint(cpu, bp);
aliguorie22a25c2009-03-12 20:12:48 +00002109 if (err) {
Anthony Liguori7267c092011-08-20 22:09:37 -05002110 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002111 return err;
2112 }
2113
Andreas Färber80b7cd72013-06-19 17:37:31 +02002114 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
aliguorie22a25c2009-03-12 20:12:48 +00002115 } else {
2116 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002117 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002118 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002119 }
aliguorie22a25c2009-03-12 20:12:48 +00002120 }
2121
Andreas Färberbdc44642013-06-24 23:50:24 +02002122 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002123 err = kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002124 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002125 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002126 }
aliguorie22a25c2009-03-12 20:12:48 +00002127 }
2128 return 0;
2129}
2130
Andreas Färber62278812013-06-27 17:12:06 +02002131int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002132 target_ulong len, int type)
2133{
2134 struct kvm_sw_breakpoint *bp;
aliguorie22a25c2009-03-12 20:12:48 +00002135 int err;
2136
2137 if (type == GDB_BREAKPOINT_SW) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002138 bp = kvm_find_sw_breakpoint(cpu, addr);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002139 if (!bp) {
aliguorie22a25c2009-03-12 20:12:48 +00002140 return -ENOENT;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002141 }
aliguorie22a25c2009-03-12 20:12:48 +00002142
2143 if (bp->use_count > 1) {
2144 bp->use_count--;
2145 return 0;
2146 }
2147
Andreas Färber80b7cd72013-06-19 17:37:31 +02002148 err = kvm_arch_remove_sw_breakpoint(cpu, bp);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002149 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002150 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002151 }
aliguorie22a25c2009-03-12 20:12:48 +00002152
Andreas Färber80b7cd72013-06-19 17:37:31 +02002153 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
Anthony Liguori7267c092011-08-20 22:09:37 -05002154 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002155 } else {
2156 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002157 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002158 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002159 }
aliguorie22a25c2009-03-12 20:12:48 +00002160 }
2161
Andreas Färberbdc44642013-06-24 23:50:24 +02002162 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002163 err = kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002164 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002165 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002166 }
aliguorie22a25c2009-03-12 20:12:48 +00002167 }
2168 return 0;
2169}
2170
Andreas Färber1d5791f2013-05-27 14:40:48 +02002171void kvm_remove_all_breakpoints(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002172{
2173 struct kvm_sw_breakpoint *bp, *next;
Andreas Färber80b7cd72013-06-19 17:37:31 +02002174 KVMState *s = cpu->kvm_state;
Chen Gangdc54e252014-07-19 09:21:46 +08002175 CPUState *tmpcpu;
aliguorie22a25c2009-03-12 20:12:48 +00002176
Blue Swirl72cf2d42009-09-12 07:36:22 +00002177 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002178 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
aliguorie22a25c2009-03-12 20:12:48 +00002179 /* Try harder to find a CPU that currently sees the breakpoint. */
Chen Gangdc54e252014-07-19 09:21:46 +08002180 CPU_FOREACH(tmpcpu) {
2181 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
aliguorie22a25c2009-03-12 20:12:48 +00002182 break;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002183 }
aliguorie22a25c2009-03-12 20:12:48 +00002184 }
2185 }
Jan Kiszka78021d62012-11-12 15:04:35 +01002186 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2187 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002188 }
2189 kvm_arch_remove_all_hw_breakpoints();
2190
Andreas Färberbdc44642013-06-24 23:50:24 +02002191 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002192 kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002193 }
aliguorie22a25c2009-03-12 20:12:48 +00002194}
2195
2196#else /* !KVM_CAP_SET_GUEST_DEBUG */
2197
Stefan Weil38e478e2013-07-25 20:50:21 +02002198int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
aliguorie22a25c2009-03-12 20:12:48 +00002199{
2200 return -EINVAL;
2201}
2202
Andreas Färber62278812013-06-27 17:12:06 +02002203int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002204 target_ulong len, int type)
2205{
2206 return -EINVAL;
2207}
2208
Andreas Färber62278812013-06-27 17:12:06 +02002209int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002210 target_ulong len, int type)
2211{
2212 return -EINVAL;
2213}
2214
Andreas Färber1d5791f2013-05-27 14:40:48 +02002215void kvm_remove_all_breakpoints(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002216{
2217}
2218#endif /* !KVM_CAP_SET_GUEST_DEBUG */
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002219
Andreas Färber491d6e82013-05-26 23:38:10 +02002220int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002221{
James Hoganaed6efb2014-06-17 23:10:31 +01002222 KVMState *s = kvm_state;
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002223 struct kvm_signal_mask *sigmask;
2224 int r;
2225
Jan Kiszkaa426e122011-01-04 09:32:13 +01002226 if (!sigset) {
Andreas Färber1bc22652012-10-31 06:06:49 +01002227 return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002228 }
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002229
Anthony Liguori7267c092011-08-20 22:09:37 -05002230 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002231
James Hoganaed6efb2014-06-17 23:10:31 +01002232 sigmask->len = s->sigmask_len;
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002233 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
Andreas Färber1bc22652012-10-31 06:06:49 +01002234 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
Anthony Liguori7267c092011-08-20 22:09:37 -05002235 g_free(sigmask);
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002236
2237 return r;
2238}
Andreas Färber290adf32013-01-17 09:30:27 +01002239int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
Jan Kiszkaa1b87fe2011-02-01 22:15:51 +01002240{
Andreas Färber20d695a2012-10-31 06:57:49 +01002241 return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
Jan Kiszkaa1b87fe2011-02-01 22:15:51 +01002242}
2243
2244int kvm_on_sigbus(int code, void *addr)
2245{
2246 return kvm_arch_on_sigbus(code, addr);
2247}
Christoffer Dall0a6a7cc2014-02-26 17:20:00 +00002248
2249int kvm_create_device(KVMState *s, uint64_t type, bool test)
2250{
2251 int ret;
2252 struct kvm_create_device create_dev;
2253
2254 create_dev.type = type;
2255 create_dev.fd = -1;
2256 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
2257
2258 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
2259 return -ENOTSUP;
2260 }
2261
2262 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
2263 if (ret) {
2264 return ret;
2265 }
2266
2267 return test ? 0 : create_dev.fd;
2268}
Cornelia Huckada41352014-05-09 10:06:46 +02002269
2270int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
2271{
2272 struct kvm_one_reg reg;
2273 int r;
2274
2275 reg.id = id;
2276 reg.addr = (uintptr_t) source;
2277 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
2278 if (r) {
2279 trace_kvm_failed_reg_set(id, strerror(r));
2280 }
2281 return r;
2282}
2283
2284int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
2285{
2286 struct kvm_one_reg reg;
2287 int r;
2288
2289 reg.id = id;
2290 reg.addr = (uintptr_t) target;
2291 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
2292 if (r) {
2293 trace_kvm_failed_reg_get(id, strerror(r));
2294 }
2295 return r;
2296}
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002297
2298static void kvm_accel_class_init(ObjectClass *oc, void *data)
2299{
2300 AccelClass *ac = ACCEL_CLASS(oc);
2301 ac->name = "KVM";
Eduardo Habkost0d15da82014-09-26 17:45:29 -03002302 ac->init_machine = kvm_init;
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002303 ac->allowed = &kvm_allowed;
2304}
2305
2306static const TypeInfo kvm_accel_type = {
2307 .name = TYPE_KVM_ACCEL,
2308 .parent = TYPE_ACCEL,
2309 .class_init = kvm_accel_class_init,
Eduardo Habkostfc020862014-09-26 17:45:32 -03002310 .instance_size = sizeof(KVMState),
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002311};
2312
2313static void kvm_type_init(void)
2314{
2315 type_register_static(&kvm_accel_type);
2316}
2317
2318type_init(kvm_type_init);