1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include "qemu/osdep.h"
16#include "qapi/error.h"
17#include "trace.h"
18#include "qemu/iov.h"
19#include "qemu/main-loop.h"
20#include "qemu/thread.h"
21#include "qemu/error-report.h"
22#include "hw/virtio/virtio-access.h"
23#include "hw/virtio/virtio-blk.h"
24#include "virtio-blk.h"
25#include "block/aio.h"
26#include "hw/virtio/virtio-bus.h"
27#include "qom/object_interfaces.h"
28
29struct VirtIOBlockDataPlane {
30 bool starting;
31 bool stopping;
32
33 VirtIOBlkConf *conf;
34 VirtIODevice *vdev;
35 QEMUBH *bh;
36 unsigned long *batch_notify_vqs;
37 bool batch_notifications;
38
39
40
41
42
43
44 IOThread *iothread;
45 AioContext *ctx;
46};
47
48
49void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq)
50{
51 if (s->batch_notifications) {
52 set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs);
53 qemu_bh_schedule(s->bh);
54 } else {
55 virtio_notify_irqfd(s->vdev, vq);
56 }
57}
58
59static void notify_guest_bh(void *opaque)
60{
61 VirtIOBlockDataPlane *s = opaque;
62 unsigned nvqs = s->conf->num_queues;
63 unsigned long bitmap[BITS_TO_LONGS(nvqs)];
64 unsigned j;
65
66 memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap));
67 memset(s->batch_notify_vqs, 0, sizeof(bitmap));
68
69 for (j = 0; j < nvqs; j += BITS_PER_LONG) {
70 unsigned long bits = bitmap[j / BITS_PER_LONG];
71
72 while (bits != 0) {
73 unsigned i = j + ctzl(bits);
74 VirtQueue *vq = virtio_get_queue(s->vdev, i);
75
76 virtio_notify_irqfd(s->vdev, vq);
77
78 bits &= bits - 1;
79 }
80 }
81}
82
83
84bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
85 VirtIOBlockDataPlane **dataplane,
86 Error **errp)
87{
88 VirtIOBlockDataPlane *s;
89 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
90 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
91
92 *dataplane = NULL;
93
94 if (conf->iothread) {
95 if (!k->set_guest_notifiers || !k->ioeventfd_assign) {
96 error_setg(errp,
97 "device is incompatible with iothread "
98 "(transport does not support notifiers)");
99 return false;
100 }
101 if (!virtio_device_ioeventfd_enabled(vdev)) {
102 error_setg(errp, "ioeventfd is required for iothread");
103 return false;
104 }
105
106
107
108
109 if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
110 error_prepend(errp, "cannot start virtio-blk dataplane: ");
111 return false;
112 }
113 }
114
115 if (!virtio_device_ioeventfd_enabled(vdev)) {
116 return false;
117 }
118
119 s = g_new0(VirtIOBlockDataPlane, 1);
120 s->vdev = vdev;
121 s->conf = conf;
122
123 if (conf->iothread) {
124 s->iothread = conf->iothread;
125 object_ref(OBJECT(s->iothread));
126 s->ctx = iothread_get_aio_context(s->iothread);
127 } else {
128 s->ctx = qemu_get_aio_context();
129 }
130 s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
131 s->batch_notify_vqs = bitmap_new(conf->num_queues);
132
133 *dataplane = s;
134
135 return true;
136}
137
138
139void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
140{
141 VirtIOBlock *vblk;
142
143 if (!s) {
144 return;
145 }
146
147 vblk = VIRTIO_BLK(s->vdev);
148 assert(!vblk->dataplane_started);
149 g_free(s->batch_notify_vqs);
150 qemu_bh_delete(s->bh);
151 if (s->iothread) {
152 object_unref(OBJECT(s->iothread));
153 }
154 g_free(s);
155}
156
157
158int virtio_blk_data_plane_start(VirtIODevice *vdev)
159{
160 VirtIOBlock *vblk = VIRTIO_BLK(vdev);
161 VirtIOBlockDataPlane *s = vblk->dataplane;
162 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk)));
163 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
164 AioContext *old_context;
165 unsigned i;
166 unsigned nvqs = s->conf->num_queues;
167 Error *local_err = NULL;
168 int r;
169
170 if (vblk->dataplane_started || s->starting) {
171 return 0;
172 }
173
174 s->starting = true;
175
176 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
177 s->batch_notifications = true;
178 } else {
179 s->batch_notifications = false;
180 }
181
182
183 r = k->set_guest_notifiers(qbus->parent, nvqs, true);
184 if (r != 0) {
185 error_report("virtio-blk failed to set guest notifier (%d), "
186 "ensure -accel kvm is set.", r);
187 goto fail_guest_notifiers;
188 }
189
190
191
192
193
194 memory_region_transaction_begin();
195
196
197 for (i = 0; i < nvqs; i++) {
198 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true);
199 if (r != 0) {
200 int j = i;
201
202 fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
203 while (i--) {
204 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
205 }
206
207
208
209
210
211 memory_region_transaction_commit();
212
213 while (j--) {
214 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j);
215 }
216 goto fail_host_notifiers;
217 }
218 }
219
220 memory_region_transaction_commit();
221
222
223
224
225
226
227 s->starting = false;
228 vblk->dataplane_started = true;
229 trace_virtio_blk_data_plane_start(s);
230
231 old_context = blk_get_aio_context(s->conf->conf.blk);
232 aio_context_acquire(old_context);
233 r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err);
234 aio_context_release(old_context);
235 if (r < 0) {
236 error_report_err(local_err);
237 goto fail_aio_context;
238 }
239
240
241 virtio_blk_process_queued_requests(vblk, false);
242
243
244 for (i = 0; i < nvqs; i++) {
245 VirtQueue *vq = virtio_get_queue(s->vdev, i);
246
247 event_notifier_set(virtio_queue_get_host_notifier(vq));
248 }
249
250
251 aio_context_acquire(s->ctx);
252 for (i = 0; i < nvqs; i++) {
253 VirtQueue *vq = virtio_get_queue(s->vdev, i);
254
255 virtio_queue_aio_attach_host_notifier(vq, s->ctx);
256 }
257 aio_context_release(s->ctx);
258 return 0;
259
260 fail_aio_context:
261 memory_region_transaction_begin();
262
263 for (i = 0; i < nvqs; i++) {
264 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
265 }
266
267 memory_region_transaction_commit();
268
269 for (i = 0; i < nvqs; i++) {
270 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i);
271 }
272 fail_host_notifiers:
273 k->set_guest_notifiers(qbus->parent, nvqs, false);
274 fail_guest_notifiers:
275
276
277
278
279 virtio_blk_process_queued_requests(vblk, false);
280 vblk->dataplane_disabled = true;
281 s->starting = false;
282 vblk->dataplane_started = true;
283 return -ENOSYS;
284}
285
286
287
288
289
290static void virtio_blk_data_plane_stop_bh(void *opaque)
291{
292 VirtIOBlockDataPlane *s = opaque;
293 unsigned i;
294
295 for (i = 0; i < s->conf->num_queues; i++) {
296 VirtQueue *vq = virtio_get_queue(s->vdev, i);
297
298 virtio_queue_aio_detach_host_notifier(vq, s->ctx);
299 }
300}
301
302
303void virtio_blk_data_plane_stop(VirtIODevice *vdev)
304{
305 VirtIOBlock *vblk = VIRTIO_BLK(vdev);
306 VirtIOBlockDataPlane *s = vblk->dataplane;
307 BusState *qbus = qdev_get_parent_bus(DEVICE(vblk));
308 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
309 unsigned i;
310 unsigned nvqs = s->conf->num_queues;
311
312 if (!vblk->dataplane_started || s->stopping) {
313 return;
314 }
315
316
317 if (vblk->dataplane_disabled) {
318 vblk->dataplane_disabled = false;
319 vblk->dataplane_started = false;
320 return;
321 }
322 s->stopping = true;
323 trace_virtio_blk_data_plane_stop(s);
324
325 aio_context_acquire(s->ctx);
326 aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s);
327
328
329
330 blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL);
331
332 aio_context_release(s->ctx);
333
334
335
336
337
338 memory_region_transaction_begin();
339
340 for (i = 0; i < nvqs; i++) {
341 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
342 }
343
344
345
346
347
348 memory_region_transaction_commit();
349
350 for (i = 0; i < nvqs; i++) {
351 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i);
352 }
353
354 qemu_bh_cancel(s->bh);
355 notify_guest_bh(s);
356
357
358 k->set_guest_notifiers(qbus->parent, nvqs, false);
359
360 vblk->dataplane_started = false;
361 s->stopping = false;
362}
363