Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 1 | /* |
| 2 | * QEMU System Emulator |
| 3 | * |
| 4 | * Copyright (c) 2003-2008 Fabrice Bellard |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to deal |
| 8 | * in the Software without restriction, including without limitation the rights |
| 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 | * copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in |
| 14 | * all copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 22 | * THE SOFTWARE. |
| 23 | */ |
| 24 | |
| 25 | #include "qemu-common.h" |
Paolo Bonzini | 737e150 | 2012-12-17 18:19:44 +0100 | [diff] [blame] | 26 | #include "block/aio.h" |
Stefan Hajnoczi | 9b34277 | 2013-03-07 13:41:47 +0100 | [diff] [blame] | 27 | #include "block/thread-pool.h" |
Paolo Bonzini | 1de7afc | 2012-12-17 18:20:00 +0100 | [diff] [blame] | 28 | #include "qemu/main-loop.h" |
Kevin Wolf | 9a1e948 | 2009-10-22 17:54:38 +0200 | [diff] [blame] | 29 | |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 30 | /***********************************************************/ |
| 31 | /* bottom halves (can be seen as timers which expire ASAP) */ |
| 32 | |
| 33 | struct QEMUBH { |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 34 | AioContext *ctx; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 35 | QEMUBHFunc *cb; |
| 36 | void *opaque; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 37 | QEMUBH *next; |
Stefan Weil | 9b47b17 | 2012-04-29 19:08:45 +0200 | [diff] [blame] | 38 | bool scheduled; |
| 39 | bool idle; |
| 40 | bool deleted; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 41 | }; |
| 42 | |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 43 | QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque) |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 44 | { |
| 45 | QEMUBH *bh; |
Anthony Liguori | 7267c09 | 2011-08-20 22:09:37 -0500 | [diff] [blame] | 46 | bh = g_malloc0(sizeof(QEMUBH)); |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 47 | bh->ctx = ctx; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 48 | bh->cb = cb; |
| 49 | bh->opaque = opaque; |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 50 | qemu_mutex_lock(&ctx->bh_lock); |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 51 | bh->next = ctx->first_bh; |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 52 | /* Make sure that the members are ready before putting bh into list */ |
| 53 | smp_wmb(); |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 54 | ctx->first_bh = bh; |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 55 | qemu_mutex_unlock(&ctx->bh_lock); |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 56 | return bh; |
| 57 | } |
| 58 | |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 59 | /* Multiple occurrences of aio_bh_poll cannot be called concurrently */ |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 60 | int aio_bh_poll(AioContext *ctx) |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 61 | { |
Kevin Wolf | 7887f62 | 2011-06-07 17:51:21 +0200 | [diff] [blame] | 62 | QEMUBH *bh, **bhp, *next; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 63 | int ret; |
Kevin Wolf | 648fb0e | 2011-09-01 16:16:10 +0200 | [diff] [blame] | 64 | |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 65 | ctx->walking_bh++; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 66 | |
| 67 | ret = 0; |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 68 | for (bh = ctx->first_bh; bh; bh = next) { |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 69 | /* Make sure that fetching bh happens before accessing its members */ |
| 70 | smp_read_barrier_depends(); |
Kevin Wolf | 7887f62 | 2011-06-07 17:51:21 +0200 | [diff] [blame] | 71 | next = bh->next; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 72 | if (!bh->deleted && bh->scheduled) { |
| 73 | bh->scheduled = 0; |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 74 | /* Paired with write barrier in bh schedule to ensure reading for |
| 75 | * idle & callbacks coming after bh's scheduling. |
| 76 | */ |
| 77 | smp_rmb(); |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 78 | if (!bh->idle) |
| 79 | ret = 1; |
| 80 | bh->idle = 0; |
| 81 | bh->cb(bh->opaque); |
| 82 | } |
| 83 | } |
| 84 | |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 85 | ctx->walking_bh--; |
Kevin Wolf | 648fb0e | 2011-09-01 16:16:10 +0200 | [diff] [blame] | 86 | |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 87 | /* remove deleted bhs */ |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 88 | if (!ctx->walking_bh) { |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 89 | qemu_mutex_lock(&ctx->bh_lock); |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 90 | bhp = &ctx->first_bh; |
Kevin Wolf | 648fb0e | 2011-09-01 16:16:10 +0200 | [diff] [blame] | 91 | while (*bhp) { |
| 92 | bh = *bhp; |
| 93 | if (bh->deleted) { |
| 94 | *bhp = bh->next; |
| 95 | g_free(bh); |
| 96 | } else { |
| 97 | bhp = &bh->next; |
| 98 | } |
| 99 | } |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 100 | qemu_mutex_unlock(&ctx->bh_lock); |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 101 | } |
| 102 | |
| 103 | return ret; |
| 104 | } |
| 105 | |
| 106 | void qemu_bh_schedule_idle(QEMUBH *bh) |
| 107 | { |
| 108 | if (bh->scheduled) |
| 109 | return; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 110 | bh->idle = 1; |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 111 | /* Make sure that idle & any writes needed by the callback are done |
| 112 | * before the locations are read in the aio_bh_poll. |
| 113 | */ |
| 114 | smp_wmb(); |
| 115 | bh->scheduled = 1; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 116 | } |
| 117 | |
| 118 | void qemu_bh_schedule(QEMUBH *bh) |
| 119 | { |
Stefan Hajnoczi | 924fe12 | 2014-06-03 11:21:01 +0200 | [diff] [blame] | 120 | AioContext *ctx; |
| 121 | |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 122 | if (bh->scheduled) |
| 123 | return; |
Stefan Hajnoczi | 924fe12 | 2014-06-03 11:21:01 +0200 | [diff] [blame] | 124 | ctx = bh->ctx; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 125 | bh->idle = 0; |
Stefan Hajnoczi | 924fe12 | 2014-06-03 11:21:01 +0200 | [diff] [blame] | 126 | /* Make sure that: |
| 127 | * 1. idle & any writes needed by the callback are done before the |
| 128 | * locations are read in the aio_bh_poll. |
| 129 | * 2. ctx is loaded before scheduled is set and the callback has a chance |
| 130 | * to execute. |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 131 | */ |
Stefan Hajnoczi | 924fe12 | 2014-06-03 11:21:01 +0200 | [diff] [blame] | 132 | smp_mb(); |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 133 | bh->scheduled = 1; |
Stefan Hajnoczi | 924fe12 | 2014-06-03 11:21:01 +0200 | [diff] [blame] | 134 | aio_notify(ctx); |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 135 | } |
| 136 | |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 137 | |
| 138 | /* This func is async. |
| 139 | */ |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 140 | void qemu_bh_cancel(QEMUBH *bh) |
| 141 | { |
| 142 | bh->scheduled = 0; |
| 143 | } |
| 144 | |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 145 | /* This func is async.The bottom half will do the delete action at the finial |
| 146 | * end. |
| 147 | */ |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 148 | void qemu_bh_delete(QEMUBH *bh) |
| 149 | { |
| 150 | bh->scheduled = 0; |
| 151 | bh->deleted = 1; |
| 152 | } |
| 153 | |
Paolo Bonzini | 22bfa75 | 2012-09-24 15:11:48 +0200 | [diff] [blame] | 154 | static gboolean |
| 155 | aio_ctx_prepare(GSource *source, gint *timeout) |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 156 | { |
Paolo Bonzini | 22bfa75 | 2012-09-24 15:11:48 +0200 | [diff] [blame] | 157 | AioContext *ctx = (AioContext *) source; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 158 | QEMUBH *bh; |
Alex Bligh | 533a8cf | 2013-08-21 16:02:51 +0100 | [diff] [blame] | 159 | int deadline; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 160 | |
Alex Bligh | 533a8cf | 2013-08-21 16:02:51 +0100 | [diff] [blame] | 161 | /* We assume there is no timeout already supplied */ |
| 162 | *timeout = -1; |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 163 | for (bh = ctx->first_bh; bh; bh = bh->next) { |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 164 | if (!bh->deleted && bh->scheduled) { |
| 165 | if (bh->idle) { |
| 166 | /* idle bottom halves will be polled at least |
| 167 | * every 10ms */ |
Paolo Bonzini | 22bfa75 | 2012-09-24 15:11:48 +0200 | [diff] [blame] | 168 | *timeout = 10; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 169 | } else { |
| 170 | /* non-idle bottom halves will be executed |
| 171 | * immediately */ |
| 172 | *timeout = 0; |
Paolo Bonzini | f5022a1 | 2012-11-12 13:30:10 +0100 | [diff] [blame] | 173 | return true; |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 174 | } |
| 175 | } |
| 176 | } |
Kevin Wolf | 4f999d0 | 2009-10-22 17:54:37 +0200 | [diff] [blame] | 177 | |
Alex Bligh | 533a8cf | 2013-08-21 16:02:51 +0100 | [diff] [blame] | 178 | deadline = qemu_timeout_ns_to_ms(timerlistgroup_deadline_ns(&ctx->tlg)); |
| 179 | if (deadline == 0) { |
| 180 | *timeout = 0; |
| 181 | return true; |
| 182 | } else { |
| 183 | *timeout = qemu_soonest_timeout(*timeout, deadline); |
| 184 | } |
| 185 | |
Paolo Bonzini | f5022a1 | 2012-11-12 13:30:10 +0100 | [diff] [blame] | 186 | return false; |
Paolo Bonzini | e3713e0 | 2012-09-24 14:57:41 +0200 | [diff] [blame] | 187 | } |
| 188 | |
| 189 | static gboolean |
| 190 | aio_ctx_check(GSource *source) |
| 191 | { |
| 192 | AioContext *ctx = (AioContext *) source; |
| 193 | QEMUBH *bh; |
| 194 | |
| 195 | for (bh = ctx->first_bh; bh; bh = bh->next) { |
| 196 | if (!bh->deleted && bh->scheduled) { |
| 197 | return true; |
| 198 | } |
| 199 | } |
Alex Bligh | 533a8cf | 2013-08-21 16:02:51 +0100 | [diff] [blame] | 200 | return aio_pending(ctx) || (timerlistgroup_deadline_ns(&ctx->tlg) == 0); |
Paolo Bonzini | e3713e0 | 2012-09-24 14:57:41 +0200 | [diff] [blame] | 201 | } |
| 202 | |
| 203 | static gboolean |
| 204 | aio_ctx_dispatch(GSource *source, |
| 205 | GSourceFunc callback, |
| 206 | gpointer user_data) |
| 207 | { |
| 208 | AioContext *ctx = (AioContext *) source; |
| 209 | |
| 210 | assert(callback == NULL); |
| 211 | aio_poll(ctx, false); |
| 212 | return true; |
| 213 | } |
| 214 | |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 215 | static void |
| 216 | aio_ctx_finalize(GSource *source) |
| 217 | { |
| 218 | AioContext *ctx = (AioContext *) source; |
| 219 | |
Stefan Hajnoczi | 9b34277 | 2013-03-07 13:41:47 +0100 | [diff] [blame] | 220 | thread_pool_free(ctx->thread_pool); |
Stefan Hajnoczi | f2e5dca | 2013-04-11 17:26:25 +0200 | [diff] [blame] | 221 | aio_set_event_notifier(ctx, &ctx->notifier, NULL); |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 222 | event_notifier_cleanup(&ctx->notifier); |
Stefan Hajnoczi | 98563fc | 2014-03-03 11:30:04 +0100 | [diff] [blame] | 223 | rfifolock_destroy(&ctx->lock); |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 224 | qemu_mutex_destroy(&ctx->bh_lock); |
Stefan Hajnoczi | 6b5f876 | 2013-02-20 11:28:32 +0100 | [diff] [blame] | 225 | g_array_free(ctx->pollfds, TRUE); |
Alex Bligh | dae21b9 | 2013-08-21 16:02:49 +0100 | [diff] [blame] | 226 | timerlistgroup_deinit(&ctx->tlg); |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 227 | } |
| 228 | |
Paolo Bonzini | e3713e0 | 2012-09-24 14:57:41 +0200 | [diff] [blame] | 229 | static GSourceFuncs aio_source_funcs = { |
| 230 | aio_ctx_prepare, |
| 231 | aio_ctx_check, |
| 232 | aio_ctx_dispatch, |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 233 | aio_ctx_finalize |
Paolo Bonzini | e3713e0 | 2012-09-24 14:57:41 +0200 | [diff] [blame] | 234 | }; |
| 235 | |
| 236 | GSource *aio_get_g_source(AioContext *ctx) |
| 237 | { |
| 238 | g_source_ref(&ctx->source); |
| 239 | return &ctx->source; |
| 240 | } |
Paolo Bonzini | a915f4b | 2012-09-13 12:28:51 +0200 | [diff] [blame] | 241 | |
Stefan Hajnoczi | 9b34277 | 2013-03-07 13:41:47 +0100 | [diff] [blame] | 242 | ThreadPool *aio_get_thread_pool(AioContext *ctx) |
| 243 | { |
| 244 | if (!ctx->thread_pool) { |
| 245 | ctx->thread_pool = thread_pool_new(ctx); |
| 246 | } |
| 247 | return ctx->thread_pool; |
| 248 | } |
| 249 | |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 250 | void aio_notify(AioContext *ctx) |
| 251 | { |
| 252 | event_notifier_set(&ctx->notifier); |
| 253 | } |
| 254 | |
Alex Bligh | d5541d8 | 2013-08-21 16:02:50 +0100 | [diff] [blame] | 255 | static void aio_timerlist_notify(void *opaque) |
| 256 | { |
| 257 | aio_notify(opaque); |
| 258 | } |
| 259 | |
Stefan Hajnoczi | 98563fc | 2014-03-03 11:30:04 +0100 | [diff] [blame] | 260 | static void aio_rfifolock_cb(void *opaque) |
| 261 | { |
| 262 | /* Kick owner thread in case they are blocked in aio_poll() */ |
| 263 | aio_notify(opaque); |
| 264 | } |
| 265 | |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 266 | AioContext *aio_context_new(void) |
| 267 | { |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 268 | AioContext *ctx; |
| 269 | ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext)); |
Stefan Hajnoczi | 6b5f876 | 2013-02-20 11:28:32 +0100 | [diff] [blame] | 270 | ctx->pollfds = g_array_new(FALSE, FALSE, sizeof(GPollFD)); |
Stefan Hajnoczi | 9b34277 | 2013-03-07 13:41:47 +0100 | [diff] [blame] | 271 | ctx->thread_pool = NULL; |
Liu Ping Fan | dcc772e | 2013-07-16 12:28:58 +0800 | [diff] [blame] | 272 | qemu_mutex_init(&ctx->bh_lock); |
Stefan Hajnoczi | 98563fc | 2014-03-03 11:30:04 +0100 | [diff] [blame] | 273 | rfifolock_init(&ctx->lock, aio_rfifolock_cb, ctx); |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 274 | event_notifier_init(&ctx->notifier, false); |
| 275 | aio_set_event_notifier(ctx, &ctx->notifier, |
| 276 | (EventNotifierHandler *) |
Stefan Hajnoczi | f2e5dca | 2013-04-11 17:26:25 +0200 | [diff] [blame] | 277 | event_notifier_test_and_clear); |
Alex Bligh | d5541d8 | 2013-08-21 16:02:50 +0100 | [diff] [blame] | 278 | timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx); |
Paolo Bonzini | 2f4dc3c | 2012-09-24 18:44:14 +0200 | [diff] [blame] | 279 | |
| 280 | return ctx; |
Paolo Bonzini | e3713e0 | 2012-09-24 14:57:41 +0200 | [diff] [blame] | 281 | } |
| 282 | |
| 283 | void aio_context_ref(AioContext *ctx) |
| 284 | { |
| 285 | g_source_ref(&ctx->source); |
| 286 | } |
| 287 | |
| 288 | void aio_context_unref(AioContext *ctx) |
| 289 | { |
| 290 | g_source_unref(&ctx->source); |
Paolo Bonzini | f627aab | 2012-10-29 23:45:23 +0100 | [diff] [blame] | 291 | } |
Stefan Hajnoczi | 98563fc | 2014-03-03 11:30:04 +0100 | [diff] [blame] | 292 | |
| 293 | void aio_context_acquire(AioContext *ctx) |
| 294 | { |
| 295 | rfifolock_lock(&ctx->lock); |
| 296 | } |
| 297 | |
| 298 | void aio_context_release(AioContext *ctx) |
| 299 | { |
| 300 | rfifolock_unlock(&ctx->lock); |
| 301 | } |