1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include "qemu/osdep.h"
30#include "cpu.h"
31#include <zlib.h>
32#include "qemu/cutils.h"
33#include "qemu/bitops.h"
34#include "qemu/bitmap.h"
35#include "qemu/main-loop.h"
36#include "xbzrle.h"
37#include "ram.h"
38#include "migration.h"
39#include "socket.h"
40#include "migration/register.h"
41#include "migration/misc.h"
42#include "qemu-file.h"
43#include "postcopy-ram.h"
44#include "page_cache.h"
45#include "qemu/error-report.h"
46#include "qapi/error.h"
47#include "qapi/qapi-events-migration.h"
48#include "qapi/qmp/qerror.h"
49#include "trace.h"
50#include "exec/ram_addr.h"
51#include "exec/target_page.h"
52#include "qemu/rcu_queue.h"
53#include "migration/colo.h"
54#include "block.h"
55#include "sysemu/sysemu.h"
56#include "qemu/uuid.h"
57#include "savevm.h"
58#include "qemu/iov.h"
59
60
61
62
63
64
65
66
67
68
69#define RAM_SAVE_FLAG_FULL 0x01
70#define RAM_SAVE_FLAG_ZERO 0x02
71#define RAM_SAVE_FLAG_MEM_SIZE 0x04
72#define RAM_SAVE_FLAG_PAGE 0x08
73#define RAM_SAVE_FLAG_EOS 0x10
74#define RAM_SAVE_FLAG_CONTINUE 0x20
75#define RAM_SAVE_FLAG_XBZRLE 0x40
76
77#define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
78
79static inline bool is_zero_range(uint8_t *p, uint64_t size)
80{
81 return buffer_is_zero(p, size);
82}
83
84XBZRLECacheStats xbzrle_counters;
85
86
87
88static struct {
89
90 uint8_t *encoded_buf;
91
92 uint8_t *current_buf;
93
94 PageCache *cache;
95 QemuMutex lock;
96
97 uint8_t *zero_target_page;
98
99 uint8_t *decoded_buf;
100} XBZRLE;
101
102static void XBZRLE_cache_lock(void)
103{
104 if (migrate_use_xbzrle())
105 qemu_mutex_lock(&XBZRLE.lock);
106}
107
108static void XBZRLE_cache_unlock(void)
109{
110 if (migrate_use_xbzrle())
111 qemu_mutex_unlock(&XBZRLE.lock);
112}
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127int xbzrle_cache_resize(int64_t new_size, Error **errp)
128{
129 PageCache *new_cache;
130 int64_t ret = 0;
131
132
133 if (new_size != (size_t)new_size) {
134 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
135 "exceeding address space");
136 return -1;
137 }
138
139 if (new_size == migrate_xbzrle_cache_size()) {
140
141 return 0;
142 }
143
144 XBZRLE_cache_lock();
145
146 if (XBZRLE.cache != NULL) {
147 new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
148 if (!new_cache) {
149 ret = -1;
150 goto out;
151 }
152
153 cache_fini(XBZRLE.cache);
154 XBZRLE.cache = new_cache;
155 }
156out:
157 XBZRLE_cache_unlock();
158 return ret;
159}
160
161
162#define RAMBLOCK_FOREACH_MIGRATABLE(block) \
163 INTERNAL_RAMBLOCK_FOREACH(block) \
164 if (!qemu_ram_is_migratable(block)) {} else
165
166#undef RAMBLOCK_FOREACH
167
168static void ramblock_recv_map_init(void)
169{
170 RAMBlock *rb;
171
172 RAMBLOCK_FOREACH_MIGRATABLE(rb) {
173 assert(!rb->receivedmap);
174 rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
175 }
176}
177
178int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
179{
180 return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
181 rb->receivedmap);
182}
183
184bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
185{
186 return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
187}
188
189void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
190{
191 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
192}
193
194void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
195 size_t nr)
196{
197 bitmap_set_atomic(rb->receivedmap,
198 ramblock_recv_bitmap_offset(host_addr, rb),
199 nr);
200}
201
202#define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
203
204
205
206
207
208
209int64_t ramblock_recv_bitmap_send(QEMUFile *file,
210 const char *block_name)
211{
212 RAMBlock *block = qemu_ram_block_by_name(block_name);
213 unsigned long *le_bitmap, nbits;
214 uint64_t size;
215
216 if (!block) {
217 error_report("%s: invalid block name: %s", __func__, block_name);
218 return -1;
219 }
220
221 nbits = block->used_length >> TARGET_PAGE_BITS;
222
223
224
225
226
227
228 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
229
230
231
232
233
234
235 bitmap_to_le(le_bitmap, block->receivedmap, nbits);
236
237
238 size = DIV_ROUND_UP(nbits, 8);
239
240
241
242
243
244
245
246 size = ROUND_UP(size, 8);
247
248 qemu_put_be64(file, size);
249 qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
250
251
252
253
254 qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
255 qemu_fflush(file);
256
257 g_free(le_bitmap);
258
259 if (qemu_file_get_error(file)) {
260 return qemu_file_get_error(file);
261 }
262
263 return size + sizeof(size);
264}
265
266
267
268
269
270struct RAMSrcPageRequest {
271 RAMBlock *rb;
272 hwaddr offset;
273 hwaddr len;
274
275 QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
276};
277
278
279struct RAMState {
280
281 QEMUFile *f;
282
283 RAMBlock *last_seen_block;
284
285 RAMBlock *last_sent_block;
286
287 ram_addr_t last_page;
288
289 uint32_t last_version;
290
291 bool ram_bulk_stage;
292
293 int dirty_rate_high_cnt;
294
295
296 int64_t time_last_bitmap_sync;
297
298 uint64_t bytes_xfer_prev;
299
300 uint64_t num_dirty_pages_period;
301
302 uint64_t xbzrle_cache_miss_prev;
303
304 uint64_t iterations_prev;
305
306 uint64_t iterations;
307
308 uint64_t migration_dirty_pages;
309
310 QemuMutex bitmap_mutex;
311
312 RAMBlock *last_req_rb;
313
314 QemuMutex src_page_req_mutex;
315 QSIMPLEQ_HEAD(src_page_requests, RAMSrcPageRequest) src_page_requests;
316};
317typedef struct RAMState RAMState;
318
319static RAMState *ram_state;
320
321uint64_t ram_bytes_remaining(void)
322{
323 return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
324 0;
325}
326
327MigrationStats ram_counters;
328
329
330struct PageSearchStatus {
331
332 RAMBlock *block;
333
334 unsigned long page;
335
336 bool complete_round;
337};
338typedef struct PageSearchStatus PageSearchStatus;
339
340struct CompressParam {
341 bool done;
342 bool quit;
343 QEMUFile *file;
344 QemuMutex mutex;
345 QemuCond cond;
346 RAMBlock *block;
347 ram_addr_t offset;
348
349
350 z_stream stream;
351 uint8_t *originbuf;
352};
353typedef struct CompressParam CompressParam;
354
355struct DecompressParam {
356 bool done;
357 bool quit;
358 QemuMutex mutex;
359 QemuCond cond;
360 void *des;
361 uint8_t *compbuf;
362 int len;
363 z_stream stream;
364};
365typedef struct DecompressParam DecompressParam;
366
367static CompressParam *comp_param;
368static QemuThread *compress_threads;
369
370
371
372
373static QemuMutex comp_done_lock;
374static QemuCond comp_done_cond;
375
376static const QEMUFileOps empty_ops = { };
377
378static QEMUFile *decomp_file;
379static DecompressParam *decomp_param;
380static QemuThread *decompress_threads;
381static QemuMutex decomp_done_lock;
382static QemuCond decomp_done_cond;
383
384static int do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
385 ram_addr_t offset, uint8_t *source_buf);
386
387static void *do_data_compress(void *opaque)
388{
389 CompressParam *param = opaque;
390 RAMBlock *block;
391 ram_addr_t offset;
392
393 qemu_mutex_lock(¶m->mutex);
394 while (!param->quit) {
395 if (param->block) {
396 block = param->block;
397 offset = param->offset;
398 param->block = NULL;
399 qemu_mutex_unlock(¶m->mutex);
400
401 do_compress_ram_page(param->file, ¶m->stream, block, offset,
402 param->originbuf);
403
404 qemu_mutex_lock(&comp_done_lock);
405 param->done = true;
406 qemu_cond_signal(&comp_done_cond);
407 qemu_mutex_unlock(&comp_done_lock);
408
409 qemu_mutex_lock(¶m->mutex);
410 } else {
411 qemu_cond_wait(¶m->cond, ¶m->mutex);
412 }
413 }
414 qemu_mutex_unlock(¶m->mutex);
415
416 return NULL;
417}
418
419static inline void terminate_compression_threads(void)
420{
421 int idx, thread_count;
422
423 thread_count = migrate_compress_threads();
424
425 for (idx = 0; idx < thread_count; idx++) {
426 qemu_mutex_lock(&comp_param[idx].mutex);
427 comp_param[idx].quit = true;
428 qemu_cond_signal(&comp_param[idx].cond);
429 qemu_mutex_unlock(&comp_param[idx].mutex);
430 }
431}
432
433static void compress_threads_save_cleanup(void)
434{
435 int i, thread_count;
436
437 if (!migrate_use_compression()) {
438 return;
439 }
440 terminate_compression_threads();
441 thread_count = migrate_compress_threads();
442 for (i = 0; i < thread_count; i++) {
443
444
445
446
447 if (!comp_param[i].file) {
448 break;
449 }
450 qemu_thread_join(compress_threads + i);
451 qemu_mutex_destroy(&comp_param[i].mutex);
452 qemu_cond_destroy(&comp_param[i].cond);
453 deflateEnd(&comp_param[i].stream);
454 g_free(comp_param[i].originbuf);
455 qemu_fclose(comp_param[i].file);
456 comp_param[i].file = NULL;
457 }
458 qemu_mutex_destroy(&comp_done_lock);
459 qemu_cond_destroy(&comp_done_cond);
460 g_free(compress_threads);
461 g_free(comp_param);
462 compress_threads = NULL;
463 comp_param = NULL;
464}
465
466static int compress_threads_save_setup(void)
467{
468 int i, thread_count;
469
470 if (!migrate_use_compression()) {
471 return 0;
472 }
473 thread_count = migrate_compress_threads();
474 compress_threads = g_new0(QemuThread, thread_count);
475 comp_param = g_new0(CompressParam, thread_count);
476 qemu_cond_init(&comp_done_cond);
477 qemu_mutex_init(&comp_done_lock);
478 for (i = 0; i < thread_count; i++) {
479 comp_param[i].originbuf = g_try_malloc(TARGET_PAGE_SIZE);
480 if (!comp_param[i].originbuf) {
481 goto exit;
482 }
483
484 if (deflateInit(&comp_param[i].stream,
485 migrate_compress_level()) != Z_OK) {
486 g_free(comp_param[i].originbuf);
487 goto exit;
488 }
489
490
491
492
493 comp_param[i].file = qemu_fopen_ops(NULL, &empty_ops);
494 comp_param[i].done = true;
495 comp_param[i].quit = false;
496 qemu_mutex_init(&comp_param[i].mutex);
497 qemu_cond_init(&comp_param[i].cond);
498 qemu_thread_create(compress_threads + i, "compress",
499 do_data_compress, comp_param + i,
500 QEMU_THREAD_JOINABLE);
501 }
502 return 0;
503
504exit:
505 compress_threads_save_cleanup();
506 return -1;
507}
508
509
510
511#define MULTIFD_MAGIC 0x11223344U
512#define MULTIFD_VERSION 1
513
514#define MULTIFD_FLAG_SYNC (1 << 0)
515
516typedef struct {
517 uint32_t magic;
518 uint32_t version;
519 unsigned char uuid[16];
520 uint8_t id;
521} __attribute__((packed)) MultiFDInit_t;
522
523typedef struct {
524 uint32_t magic;
525 uint32_t version;
526 uint32_t flags;
527 uint32_t size;
528 uint32_t used;
529 uint64_t packet_num;
530 char ramblock[256];
531 uint64_t offset[];
532} __attribute__((packed)) MultiFDPacket_t;
533
534typedef struct {
535
536 uint32_t used;
537
538 uint32_t allocated;
539
540 uint64_t packet_num;
541
542 ram_addr_t *offset;
543
544 struct iovec *iov;
545 RAMBlock *block;
546} MultiFDPages_t;
547
548typedef struct {
549
550
551 uint8_t id;
552
553 char *name;
554
555 QemuThread thread;
556
557 QIOChannel *c;
558
559 QemuSemaphore sem;
560
561 QemuMutex mutex;
562
563 bool running;
564
565 bool quit;
566
567 int pending_job;
568
569 MultiFDPages_t *pages;
570
571 uint32_t packet_len;
572
573 MultiFDPacket_t *packet;
574
575 uint32_t flags;
576
577 uint64_t packet_num;
578
579
580 uint64_t num_packets;
581
582 uint64_t num_pages;
583
584 QemuSemaphore sem_sync;
585} MultiFDSendParams;
586
587typedef struct {
588
589
590 uint8_t id;
591
592 char *name;
593
594 QemuThread thread;
595
596 QIOChannel *c;
597
598 QemuMutex mutex;
599
600 bool running;
601
602 MultiFDPages_t *pages;
603
604 uint32_t packet_len;
605
606 MultiFDPacket_t *packet;
607
608 uint32_t flags;
609
610 uint64_t packet_num;
611
612
613 uint64_t num_packets;
614
615 uint64_t num_pages;
616
617 QemuSemaphore sem_sync;
618} MultiFDRecvParams;
619
620static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp)
621{
622 MultiFDInit_t msg;
623 int ret;
624
625 msg.magic = cpu_to_be32(MULTIFD_MAGIC);
626 msg.version = cpu_to_be32(MULTIFD_VERSION);
627 msg.id = p->id;
628 memcpy(msg.uuid, &qemu_uuid.data, sizeof(msg.uuid));
629
630 ret = qio_channel_write_all(p->c, (char *)&msg, sizeof(msg), errp);
631 if (ret != 0) {
632 return -1;
633 }
634 return 0;
635}
636
637static int multifd_recv_initial_packet(QIOChannel *c, Error **errp)
638{
639 MultiFDInit_t msg;
640 int ret;
641
642 ret = qio_channel_read_all(c, (char *)&msg, sizeof(msg), errp);
643 if (ret != 0) {
644 return -1;
645 }
646
647 be32_to_cpus(&msg.magic);
648 be32_to_cpus(&msg.version);
649
650 if (msg.magic != MULTIFD_MAGIC) {
651 error_setg(errp, "multifd: received packet magic %x "
652 "expected %x", msg.magic, MULTIFD_MAGIC);
653 return -1;
654 }
655
656 if (msg.version != MULTIFD_VERSION) {
657 error_setg(errp, "multifd: received packet version %d "
658 "expected %d", msg.version, MULTIFD_VERSION);
659 return -1;
660 }
661
662 if (memcmp(msg.uuid, &qemu_uuid, sizeof(qemu_uuid))) {
663 char *uuid = qemu_uuid_unparse_strdup(&qemu_uuid);
664 char *msg_uuid = qemu_uuid_unparse_strdup((const QemuUUID *)msg.uuid);
665
666 error_setg(errp, "multifd: received uuid '%s' and expected "
667 "uuid '%s' for channel %hhd", msg_uuid, uuid, msg.id);
668 g_free(uuid);
669 g_free(msg_uuid);
670 return -1;
671 }
672
673 if (msg.id > migrate_multifd_channels()) {
674 error_setg(errp, "multifd: received channel version %d "
675 "expected %d", msg.version, MULTIFD_VERSION);
676 return -1;
677 }
678
679 return msg.id;
680}
681
682static MultiFDPages_t *multifd_pages_init(size_t size)
683{
684 MultiFDPages_t *pages = g_new0(MultiFDPages_t, 1);
685
686 pages->allocated = size;
687 pages->iov = g_new0(struct iovec, size);
688 pages->offset = g_new0(ram_addr_t, size);
689
690 return pages;
691}
692
693static void multifd_pages_clear(MultiFDPages_t *pages)
694{
695 pages->used = 0;
696 pages->allocated = 0;
697 pages->packet_num = 0;
698 pages->block = NULL;
699 g_free(pages->iov);
700 pages->iov = NULL;
701 g_free(pages->offset);
702 pages->offset = NULL;
703 g_free(pages);
704}
705
706static void multifd_send_fill_packet(MultiFDSendParams *p)
707{
708 MultiFDPacket_t *packet = p->packet;
709 int i;
710
711 packet->magic = cpu_to_be32(MULTIFD_MAGIC);
712 packet->version = cpu_to_be32(MULTIFD_VERSION);
713 packet->flags = cpu_to_be32(p->flags);
714 packet->size = cpu_to_be32(migrate_multifd_page_count());
715 packet->used = cpu_to_be32(p->pages->used);
716 packet->packet_num = cpu_to_be64(p->packet_num);
717
718 if (p->pages->block) {
719 strncpy(packet->ramblock, p->pages->block->idstr, 256);
720 }
721
722 for (i = 0; i < p->pages->used; i++) {
723 packet->offset[i] = cpu_to_be64(p->pages->offset[i]);
724 }
725}
726
727static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp)
728{
729 MultiFDPacket_t *packet = p->packet;
730 RAMBlock *block;
731 int i;
732
733 be32_to_cpus(&packet->magic);
734 if (packet->magic != MULTIFD_MAGIC) {
735 error_setg(errp, "multifd: received packet "
736 "magic %x and expected magic %x",
737 packet->magic, MULTIFD_MAGIC);
738 return -1;
739 }
740
741 be32_to_cpus(&packet->version);
742 if (packet->version != MULTIFD_VERSION) {
743 error_setg(errp, "multifd: received packet "
744 "version %d and expected version %d",
745 packet->version, MULTIFD_VERSION);
746 return -1;
747 }
748
749 p->flags = be32_to_cpu(packet->flags);
750
751 be32_to_cpus(&packet->size);
752 if (packet->size > migrate_multifd_page_count()) {
753 error_setg(errp, "multifd: received packet "
754 "with size %d and expected maximum size %d",
755 packet->size, migrate_multifd_page_count()) ;
756 return -1;
757 }
758
759 p->pages->used = be32_to_cpu(packet->used);
760 if (p->pages->used > packet->size) {
761 error_setg(errp, "multifd: received packet "
762 "with size %d and expected maximum size %d",
763 p->pages->used, packet->size) ;
764 return -1;
765 }
766
767 p->packet_num = be64_to_cpu(packet->packet_num);
768
769 if (p->pages->used) {
770
771 packet->ramblock[255] = 0;
772 block = qemu_ram_block_by_name(packet->ramblock);
773 if (!block) {
774 error_setg(errp, "multifd: unknown ram block %s",
775 packet->ramblock);
776 return -1;
777 }
778 }
779
780 for (i = 0; i < p->pages->used; i++) {
781 ram_addr_t offset = be64_to_cpu(packet->offset[i]);
782
783 if (offset > (block->used_length - TARGET_PAGE_SIZE)) {
784 error_setg(errp, "multifd: offset too long " RAM_ADDR_FMT
785 " (max " RAM_ADDR_FMT ")",
786 offset, block->max_length);
787 return -1;
788 }
789 p->pages->iov[i].iov_base = block->host + offset;
790 p->pages->iov[i].iov_len = TARGET_PAGE_SIZE;
791 }
792
793 return 0;
794}
795
796struct {
797 MultiFDSendParams *params;
798
799 int count;
800
801 MultiFDPages_t *pages;
802
803 QemuSemaphore sem_sync;
804
805 uint64_t packet_num;
806
807 QemuSemaphore channels_ready;
808} *multifd_send_state;
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828static void multifd_send_pages(void)
829{
830 int i;
831 static int next_channel;
832 MultiFDSendParams *p = NULL;
833 MultiFDPages_t *pages = multifd_send_state->pages;
834 uint64_t transferred;
835
836 qemu_sem_wait(&multifd_send_state->channels_ready);
837 for (i = next_channel;; i = (i + 1) % migrate_multifd_channels()) {
838 p = &multifd_send_state->params[i];
839
840 qemu_mutex_lock(&p->mutex);
841 if (!p->pending_job) {
842 p->pending_job++;
843 next_channel = (i + 1) % migrate_multifd_channels();
844 break;
845 }
846 qemu_mutex_unlock(&p->mutex);
847 }
848 p->pages->used = 0;
849
850 p->packet_num = multifd_send_state->packet_num++;
851 p->pages->block = NULL;
852 multifd_send_state->pages = p->pages;
853 p->pages = pages;
854 transferred = ((uint64_t) pages->used) * TARGET_PAGE_SIZE + p->packet_len;
855 ram_counters.multifd_bytes += transferred;
856 ram_counters.transferred += transferred;;
857 qemu_mutex_unlock(&p->mutex);
858 qemu_sem_post(&p->sem);
859}
860
861static void multifd_queue_page(RAMBlock *block, ram_addr_t offset)
862{
863 MultiFDPages_t *pages = multifd_send_state->pages;
864
865 if (!pages->block) {
866 pages->block = block;
867 }
868
869 if (pages->block == block) {
870 pages->offset[pages->used] = offset;
871 pages->iov[pages->used].iov_base = block->host + offset;
872 pages->iov[pages->used].iov_len = TARGET_PAGE_SIZE;
873 pages->used++;
874
875 if (pages->used < pages->allocated) {
876 return;
877 }
878 }
879
880 multifd_send_pages();
881
882 if (pages->block != block) {
883 multifd_queue_page(block, offset);
884 }
885}
886
887static void multifd_send_terminate_threads(Error *err)
888{
889 int i;
890
891 if (err) {
892 MigrationState *s = migrate_get_current();
893 migrate_set_error(s, err);
894 if (s->state == MIGRATION_STATUS_SETUP ||
895 s->state == MIGRATION_STATUS_PRE_SWITCHOVER ||
896 s->state == MIGRATION_STATUS_DEVICE ||
897 s->state == MIGRATION_STATUS_ACTIVE) {
898 migrate_set_state(&s->state, s->state,
899 MIGRATION_STATUS_FAILED);
900 }
901 }
902
903 for (i = 0; i < migrate_multifd_channels(); i++) {
904 MultiFDSendParams *p = &multifd_send_state->params[i];
905
906 qemu_mutex_lock(&p->mutex);
907 p->quit = true;
908 qemu_sem_post(&p->sem);
909 qemu_mutex_unlock(&p->mutex);
910 }
911}
912
913int multifd_save_cleanup(Error **errp)
914{
915 int i;
916 int ret = 0;
917
918 if (!migrate_use_multifd()) {
919 return 0;
920 }
921 multifd_send_terminate_threads(NULL);
922 for (i = 0; i < migrate_multifd_channels(); i++) {
923 MultiFDSendParams *p = &multifd_send_state->params[i];
924
925 if (p->running) {
926 qemu_thread_join(&p->thread);
927 }
928 socket_send_channel_destroy(p->c);
929 p->c = NULL;
930 qemu_mutex_destroy(&p->mutex);
931 qemu_sem_destroy(&p->sem);
932 qemu_sem_destroy(&p->sem_sync);
933 g_free(p->name);
934 p->name = NULL;
935 multifd_pages_clear(p->pages);
936 p->pages = NULL;
937 p->packet_len = 0;
938 g_free(p->packet);
939 p->packet = NULL;
940 }
941 qemu_sem_destroy(&multifd_send_state->channels_ready);
942 qemu_sem_destroy(&multifd_send_state->sem_sync);
943 g_free(multifd_send_state->params);
944 multifd_send_state->params = NULL;
945 multifd_pages_clear(multifd_send_state->pages);
946 multifd_send_state->pages = NULL;
947 g_free(multifd_send_state);
948 multifd_send_state = NULL;
949 return ret;
950}
951
952static void multifd_send_sync_main(void)
953{
954 int i;
955
956 if (!migrate_use_multifd()) {
957 return;
958 }
959 if (multifd_send_state->pages->used) {
960 multifd_send_pages();
961 }
962 for (i = 0; i < migrate_multifd_channels(); i++) {
963 MultiFDSendParams *p = &multifd_send_state->params[i];
964
965 trace_multifd_send_sync_main_signal(p->id);
966
967 qemu_mutex_lock(&p->mutex);
968
969 p->packet_num = multifd_send_state->packet_num++;
970 p->flags |= MULTIFD_FLAG_SYNC;
971 p->pending_job++;
972 qemu_mutex_unlock(&p->mutex);
973 qemu_sem_post(&p->sem);
974 }
975 for (i = 0; i < migrate_multifd_channels(); i++) {
976 MultiFDSendParams *p = &multifd_send_state->params[i];
977
978 trace_multifd_send_sync_main_wait(p->id);
979 qemu_sem_wait(&multifd_send_state->sem_sync);
980 }
981 trace_multifd_send_sync_main(multifd_send_state->packet_num);
982}
983
984static void *multifd_send_thread(void *opaque)
985{
986 MultiFDSendParams *p = opaque;
987 Error *local_err = NULL;
988 int ret;
989
990 trace_multifd_send_thread_start(p->id);
991
992 if (multifd_send_initial_packet(p, &local_err) < 0) {
993 goto out;
994 }
995
996 p->num_packets = 1;
997
998 while (true) {
999 qemu_sem_wait(&p->sem);
1000 qemu_mutex_lock(&p->mutex);
1001
1002 if (p->pending_job) {
1003 uint32_t used = p->pages->used;
1004 uint64_t packet_num = p->packet_num;
1005 uint32_t flags = p->flags;
1006
1007 multifd_send_fill_packet(p);
1008 p->flags = 0;
1009 p->num_packets++;
1010 p->num_pages += used;
1011 p->pages->used = 0;
1012 qemu_mutex_unlock(&p->mutex);
1013
1014 trace_multifd_send(p->id, packet_num, used, flags);
1015
1016 ret = qio_channel_write_all(p->c, (void *)p->packet,
1017 p->packet_len, &local_err);
1018 if (ret != 0) {
1019 break;
1020 }
1021
1022 ret = qio_channel_writev_all(p->c, p->pages->iov, used, &local_err);
1023 if (ret != 0) {
1024 break;
1025 }
1026
1027 qemu_mutex_lock(&p->mutex);
1028 p->pending_job--;
1029 qemu_mutex_unlock(&p->mutex);
1030
1031 if (flags & MULTIFD_FLAG_SYNC) {
1032 qemu_sem_post(&multifd_send_state->sem_sync);
1033 }
1034 qemu_sem_post(&multifd_send_state->channels_ready);
1035 } else if (p->quit) {
1036 qemu_mutex_unlock(&p->mutex);
1037 break;
1038 } else {
1039 qemu_mutex_unlock(&p->mutex);
1040
1041 }
1042 }
1043
1044out:
1045 if (local_err) {
1046 multifd_send_terminate_threads(local_err);
1047 }
1048
1049 qemu_mutex_lock(&p->mutex);
1050 p->running = false;
1051 qemu_mutex_unlock(&p->mutex);
1052
1053 trace_multifd_send_thread_end(p->id, p->num_packets, p->num_pages);
1054
1055 return NULL;
1056}
1057
1058static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque)
1059{
1060 MultiFDSendParams *p = opaque;
1061 QIOChannel *sioc = QIO_CHANNEL(qio_task_get_source(task));
1062 Error *local_err = NULL;
1063
1064 if (qio_task_propagate_error(task, &local_err)) {
1065 if (multifd_save_cleanup(&local_err) != 0) {
1066 migrate_set_error(migrate_get_current(), local_err);
1067 }
1068 } else {
1069 p->c = QIO_CHANNEL(sioc);
1070 qio_channel_set_delay(p->c, false);
1071 p->running = true;
1072 qemu_thread_create(&p->thread, p->name, multifd_send_thread, p,
1073 QEMU_THREAD_JOINABLE);
1074
1075 atomic_inc(&multifd_send_state->count);
1076 }
1077}
1078
1079int multifd_save_setup(void)
1080{
1081 int thread_count;
1082 uint32_t page_count = migrate_multifd_page_count();
1083 uint8_t i;
1084
1085 if (!migrate_use_multifd()) {
1086 return 0;
1087 }
1088 thread_count = migrate_multifd_channels();
1089 multifd_send_state = g_malloc0(sizeof(*multifd_send_state));
1090 multifd_send_state->params = g_new0(MultiFDSendParams, thread_count);
1091 atomic_set(&multifd_send_state->count, 0);
1092 multifd_send_state->pages = multifd_pages_init(page_count);
1093 qemu_sem_init(&multifd_send_state->sem_sync, 0);
1094 qemu_sem_init(&multifd_send_state->channels_ready, 0);
1095
1096 for (i = 0; i < thread_count; i++) {
1097 MultiFDSendParams *p = &multifd_send_state->params[i];
1098
1099 qemu_mutex_init(&p->mutex);
1100 qemu_sem_init(&p->sem, 0);
1101 qemu_sem_init(&p->sem_sync, 0);
1102 p->quit = false;
1103 p->pending_job = 0;
1104 p->id = i;
1105 p->pages = multifd_pages_init(page_count);
1106 p->packet_len = sizeof(MultiFDPacket_t)
1107 + sizeof(ram_addr_t) * page_count;
1108 p->packet = g_malloc0(p->packet_len);
1109 p->name = g_strdup_printf("multifdsend_%d", i);
1110 socket_send_channel_create(multifd_new_send_channel_async, p);
1111 }
1112 return 0;
1113}
1114
1115struct {
1116 MultiFDRecvParams *params;
1117
1118 int count;
1119
1120 QemuSemaphore sem_sync;
1121
1122 uint64_t packet_num;
1123} *multifd_recv_state;
1124
1125static void multifd_recv_terminate_threads(Error *err)
1126{
1127 int i;
1128
1129 if (err) {
1130 MigrationState *s = migrate_get_current();
1131 migrate_set_error(s, err);
1132 if (s->state == MIGRATION_STATUS_SETUP ||
1133 s->state == MIGRATION_STATUS_ACTIVE) {
1134 migrate_set_state(&s->state, s->state,
1135 MIGRATION_STATUS_FAILED);
1136 }
1137 }
1138
1139 for (i = 0; i < migrate_multifd_channels(); i++) {
1140 MultiFDRecvParams *p = &multifd_recv_state->params[i];
1141
1142 qemu_mutex_lock(&p->mutex);
1143
1144
1145
1146
1147 qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
1148 qemu_mutex_unlock(&p->mutex);
1149 }
1150}
1151
1152int multifd_load_cleanup(Error **errp)
1153{
1154 int i;
1155 int ret = 0;
1156
1157 if (!migrate_use_multifd()) {
1158 return 0;
1159 }
1160 multifd_recv_terminate_threads(NULL);
1161 for (i = 0; i < migrate_multifd_channels(); i++) {
1162 MultiFDRecvParams *p = &multifd_recv_state->params[i];
1163
1164 if (p->running) {
1165 qemu_thread_join(&p->thread);
1166 }
1167 object_unref(OBJECT(p->c));
1168 p->c = NULL;
1169 qemu_mutex_destroy(&p->mutex);
1170 qemu_sem_destroy(&p->sem_sync);
1171 g_free(p->name);
1172 p->name = NULL;
1173 multifd_pages_clear(p->pages);
1174 p->pages = NULL;
1175 p->packet_len = 0;
1176 g_free(p->packet);
1177 p->packet = NULL;
1178 }
1179 qemu_sem_destroy(&multifd_recv_state->sem_sync);
1180 g_free(multifd_recv_state->params);
1181 multifd_recv_state->params = NULL;
1182 g_free(multifd_recv_state);
1183 multifd_recv_state = NULL;
1184
1185 return ret;
1186}
1187
1188static void multifd_recv_sync_main(void)
1189{
1190 int i;
1191
1192 if (!migrate_use_multifd()) {
1193 return;
1194 }
1195 for (i = 0; i < migrate_multifd_channels(); i++) {
1196 MultiFDRecvParams *p = &multifd_recv_state->params[i];
1197
1198 trace_multifd_recv_sync_main_wait(p->id);
1199 qemu_sem_wait(&multifd_recv_state->sem_sync);
1200 qemu_mutex_lock(&p->mutex);
1201 if (multifd_recv_state->packet_num < p->packet_num) {
1202 multifd_recv_state->packet_num = p->packet_num;
1203 }
1204 qemu_mutex_unlock(&p->mutex);
1205 }
1206 for (i = 0; i < migrate_multifd_channels(); i++) {
1207 MultiFDRecvParams *p = &multifd_recv_state->params[i];
1208
1209 trace_multifd_recv_sync_main_signal(p->id);
1210 qemu_sem_post(&p->sem_sync);
1211 }
1212 trace_multifd_recv_sync_main(multifd_recv_state->packet_num);
1213}
1214
1215static void *multifd_recv_thread(void *opaque)
1216{
1217 MultiFDRecvParams *p = opaque;
1218 Error *local_err = NULL;
1219 int ret;
1220
1221 trace_multifd_recv_thread_start(p->id);
1222
1223 while (true) {
1224 uint32_t used;
1225 uint32_t flags;
1226
1227 ret = qio_channel_read_all_eof(p->c, (void *)p->packet,
1228 p->packet_len, &local_err);
1229 if (ret == 0) {
1230 break;
1231 }
1232 if (ret == -1) {
1233 break;
1234 }
1235
1236 qemu_mutex_lock(&p->mutex);
1237 ret = multifd_recv_unfill_packet(p, &local_err);
1238 if (ret) {
1239 qemu_mutex_unlock(&p->mutex);
1240 break;
1241 }
1242
1243 used = p->pages->used;
1244 flags = p->flags;
1245 trace_multifd_recv(p->id, p->packet_num, used, flags);
1246 p->num_packets++;
1247 p->num_pages += used;
1248 qemu_mutex_unlock(&p->mutex);
1249
1250 ret = qio_channel_readv_all(p->c, p->pages->iov, used, &local_err);
1251 if (ret != 0) {
1252 break;
1253 }
1254
1255 if (flags & MULTIFD_FLAG_SYNC) {
1256 qemu_sem_post(&multifd_recv_state->sem_sync);
1257 qemu_sem_wait(&p->sem_sync);
1258 }
1259 }
1260
1261 if (local_err) {
1262 multifd_recv_terminate_threads(local_err);
1263 }
1264 qemu_mutex_lock(&p->mutex);
1265 p->running = false;
1266 qemu_mutex_unlock(&p->mutex);
1267
1268 trace_multifd_recv_thread_end(p->id, p->num_packets, p->num_pages);
1269
1270 return NULL;
1271}
1272
1273int multifd_load_setup(void)
1274{
1275 int thread_count;
1276 uint32_t page_count = migrate_multifd_page_count();
1277 uint8_t i;
1278
1279 if (!migrate_use_multifd()) {
1280 return 0;
1281 }
1282 thread_count = migrate_multifd_channels();
1283 multifd_recv_state = g_malloc0(sizeof(*multifd_recv_state));
1284 multifd_recv_state->params = g_new0(MultiFDRecvParams, thread_count);
1285 atomic_set(&multifd_recv_state->count, 0);
1286 qemu_sem_init(&multifd_recv_state->sem_sync, 0);
1287
1288 for (i = 0; i < thread_count; i++) {
1289 MultiFDRecvParams *p = &multifd_recv_state->params[i];
1290
1291 qemu_mutex_init(&p->mutex);
1292 qemu_sem_init(&p->sem_sync, 0);
1293 p->id = i;
1294 p->pages = multifd_pages_init(page_count);
1295 p->packet_len = sizeof(MultiFDPacket_t)
1296 + sizeof(ram_addr_t) * page_count;
1297 p->packet = g_malloc0(p->packet_len);
1298 p->name = g_strdup_printf("multifdrecv_%d", i);
1299 }
1300 return 0;
1301}
1302
1303bool multifd_recv_all_channels_created(void)
1304{
1305 int thread_count = migrate_multifd_channels();
1306
1307 if (!migrate_use_multifd()) {
1308 return true;
1309 }
1310
1311 return thread_count == atomic_read(&multifd_recv_state->count);
1312}
1313
1314
1315bool multifd_recv_new_channel(QIOChannel *ioc)
1316{
1317 MultiFDRecvParams *p;
1318 Error *local_err = NULL;
1319 int id;
1320
1321 id = multifd_recv_initial_packet(ioc, &local_err);
1322 if (id < 0) {
1323 multifd_recv_terminate_threads(local_err);
1324 return false;
1325 }
1326
1327 p = &multifd_recv_state->params[id];
1328 if (p->c != NULL) {
1329 error_setg(&local_err, "multifd: received id '%d' already setup'",
1330 id);
1331 multifd_recv_terminate_threads(local_err);
1332 return false;
1333 }
1334 p->c = ioc;
1335 object_ref(OBJECT(ioc));
1336
1337 p->num_packets = 1;
1338
1339 p->running = true;
1340 qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p,
1341 QEMU_THREAD_JOINABLE);
1342 atomic_inc(&multifd_recv_state->count);
1343 return multifd_recv_state->count == migrate_multifd_channels();
1344}
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358static size_t save_page_header(RAMState *rs, QEMUFile *f, RAMBlock *block,
1359 ram_addr_t offset)
1360{
1361 size_t size, len;
1362
1363 if (block == rs->last_sent_block) {
1364 offset |= RAM_SAVE_FLAG_CONTINUE;
1365 }
1366 qemu_put_be64(f, offset);
1367 size = 8;
1368
1369 if (!(offset & RAM_SAVE_FLAG_CONTINUE)) {
1370 len = strlen(block->idstr);
1371 qemu_put_byte(f, len);
1372 qemu_put_buffer(f, (uint8_t *)block->idstr, len);
1373 size += 1 + len;
1374 rs->last_sent_block = block;
1375 }
1376 return size;
1377}
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388static void mig_throttle_guest_down(void)
1389{
1390 MigrationState *s = migrate_get_current();
1391 uint64_t pct_initial = s->parameters.cpu_throttle_initial;
1392 uint64_t pct_icrement = s->parameters.cpu_throttle_increment;
1393
1394
1395 if (!cpu_throttle_active()) {
1396 cpu_throttle_set(pct_initial);
1397 } else {
1398
1399 cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement);
1400 }
1401}
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415static void xbzrle_cache_zero_page(RAMState *rs, ram_addr_t current_addr)
1416{
1417 if (rs->ram_bulk_stage || !migrate_use_xbzrle()) {
1418 return;
1419 }
1420
1421
1422
1423 cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
1424 ram_counters.dirty_sync_count);
1425}
1426
1427#define ENCODING_FLAG_XBZRLE 0x1
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443static int save_xbzrle_page(RAMState *rs, uint8_t **current_data,
1444 ram_addr_t current_addr, RAMBlock *block,
1445 ram_addr_t offset, bool last_stage)
1446{
1447 int encoded_len = 0, bytes_xbzrle;
1448 uint8_t *prev_cached_page;
1449
1450 if (!cache_is_cached(XBZRLE.cache, current_addr,
1451 ram_counters.dirty_sync_count)) {
1452 xbzrle_counters.cache_miss++;
1453 if (!last_stage) {
1454 if (cache_insert(XBZRLE.cache, current_addr, *current_data,
1455 ram_counters.dirty_sync_count) == -1) {
1456 return -1;
1457 } else {
1458
1459
1460 *current_data = get_cached_data(XBZRLE.cache, current_addr);
1461 }
1462 }
1463 return -1;
1464 }
1465
1466 prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
1467
1468
1469 memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
1470
1471
1472 encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
1473 TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
1474 TARGET_PAGE_SIZE);
1475 if (encoded_len == 0) {
1476 trace_save_xbzrle_page_skipping();
1477 return 0;
1478 } else if (encoded_len == -1) {
1479 trace_save_xbzrle_page_overflow();
1480 xbzrle_counters.overflow++;
1481
1482 if (!last_stage) {
1483 memcpy(prev_cached_page, *current_data, TARGET_PAGE_SIZE);
1484 *current_data = prev_cached_page;
1485 }
1486 return -1;
1487 }
1488
1489
1490 if (!last_stage) {
1491 memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
1492 }
1493
1494
1495 bytes_xbzrle = save_page_header(rs, rs->f, block,
1496 offset | RAM_SAVE_FLAG_XBZRLE);
1497 qemu_put_byte(rs->f, ENCODING_FLAG_XBZRLE);
1498 qemu_put_be16(rs->f, encoded_len);
1499 qemu_put_buffer(rs->f, XBZRLE.encoded_buf, encoded_len);
1500 bytes_xbzrle += encoded_len + 1 + 2;
1501 xbzrle_counters.pages++;
1502 xbzrle_counters.bytes += bytes_xbzrle;
1503 ram_counters.transferred += bytes_xbzrle;
1504
1505 return 1;
1506}
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519static inline
1520unsigned long migration_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
1521 unsigned long start)
1522{
1523 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
1524 unsigned long *bitmap = rb->bmap;
1525 unsigned long next;
1526
1527 if (!qemu_ram_is_migratable(rb)) {
1528 return size;
1529 }
1530
1531 if (rs->ram_bulk_stage && start > 0) {
1532 next = start + 1;
1533 } else {
1534 next = find_next_bit(bitmap, size, start);
1535 }
1536
1537 return next;
1538}
1539
1540static inline bool migration_bitmap_clear_dirty(RAMState *rs,
1541 RAMBlock *rb,
1542 unsigned long page)
1543{
1544 bool ret;
1545
1546 ret = test_and_clear_bit(page, rb->bmap);
1547
1548 if (ret) {
1549 rs->migration_dirty_pages--;
1550 }
1551 return ret;
1552}
1553
1554static void migration_bitmap_sync_range(RAMState *rs, RAMBlock *rb,
1555 ram_addr_t start, ram_addr_t length)
1556{
1557 rs->migration_dirty_pages +=
1558 cpu_physical_memory_sync_dirty_bitmap(rb, start, length,
1559 &rs->num_dirty_pages_period);
1560}
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571uint64_t ram_pagesize_summary(void)
1572{
1573 RAMBlock *block;
1574 uint64_t summary = 0;
1575
1576 RAMBLOCK_FOREACH_MIGRATABLE(block) {
1577 summary |= block->page_size;
1578 }
1579
1580 return summary;
1581}
1582
1583static void migration_update_rates(RAMState *rs, int64_t end_time)
1584{
1585 uint64_t iter_count = rs->iterations - rs->iterations_prev;
1586
1587
1588 ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000
1589 / (end_time - rs->time_last_bitmap_sync);
1590
1591 if (!iter_count) {
1592 return;
1593 }
1594
1595 if (migrate_use_xbzrle()) {
1596 xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
1597 rs->xbzrle_cache_miss_prev) / iter_count;
1598 rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
1599 }
1600}
1601
1602static void migration_bitmap_sync(RAMState *rs)
1603{
1604 RAMBlock *block;
1605 int64_t end_time;
1606 uint64_t bytes_xfer_now;
1607
1608 ram_counters.dirty_sync_count++;
1609
1610 if (!rs->time_last_bitmap_sync) {
1611 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1612 }
1613
1614 trace_migration_bitmap_sync_start();
1615 memory_global_dirty_log_sync();
1616
1617 qemu_mutex_lock(&rs->bitmap_mutex);
1618 rcu_read_lock();
1619 RAMBLOCK_FOREACH_MIGRATABLE(block) {
1620 migration_bitmap_sync_range(rs, block, 0, block->used_length);
1621 }
1622 ram_counters.remaining = ram_bytes_remaining();
1623 rcu_read_unlock();
1624 qemu_mutex_unlock(&rs->bitmap_mutex);
1625
1626 trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
1627
1628 end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1629
1630
1631 if (end_time > rs->time_last_bitmap_sync + 1000) {
1632 bytes_xfer_now = ram_counters.transferred;
1633
1634
1635
1636
1637 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
1638
1639
1640
1641
1642
1643
1644 if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE >
1645 (bytes_xfer_now - rs->bytes_xfer_prev) / 2) &&
1646 (++rs->dirty_rate_high_cnt >= 2)) {
1647 trace_migration_throttle();
1648 rs->dirty_rate_high_cnt = 0;
1649 mig_throttle_guest_down();
1650 }
1651 }
1652
1653 migration_update_rates(rs, end_time);
1654
1655 rs->iterations_prev = rs->iterations;
1656
1657
1658 rs->time_last_bitmap_sync = end_time;
1659 rs->num_dirty_pages_period = 0;
1660 rs->bytes_xfer_prev = bytes_xfer_now;
1661 }
1662 if (migrate_use_events()) {
1663 qapi_event_send_migration_pass(ram_counters.dirty_sync_count, NULL);
1664 }
1665}
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676static int save_zero_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
1677{
1678 uint8_t *p = block->host + offset;
1679 int pages = -1;
1680
1681 if (is_zero_range(p, TARGET_PAGE_SIZE)) {
1682 ram_counters.duplicate++;
1683 ram_counters.transferred +=
1684 save_page_header(rs, rs->f, block, offset | RAM_SAVE_FLAG_ZERO);
1685 qemu_put_byte(rs->f, 0);
1686 ram_counters.transferred += 1;
1687 pages = 1;
1688 }
1689
1690 return pages;
1691}
1692
1693static void ram_release_pages(const char *rbname, uint64_t offset, int pages)
1694{
1695 if (!migrate_release_ram() || !migration_in_postcopy()) {
1696 return;
1697 }
1698
1699 ram_discard_range(rbname, offset, pages << TARGET_PAGE_BITS);
1700}
1701
1702
1703
1704
1705
1706
1707
1708
1709static bool control_save_page(RAMState *rs, RAMBlock *block, ram_addr_t offset,
1710 int *pages)
1711{
1712 uint64_t bytes_xmit = 0;
1713 int ret;
1714
1715 *pages = -1;
1716 ret = ram_control_save_page(rs->f, block->offset, offset, TARGET_PAGE_SIZE,
1717 &bytes_xmit);
1718 if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
1719 return false;
1720 }
1721
1722 if (bytes_xmit) {
1723 ram_counters.transferred += bytes_xmit;
1724 *pages = 1;
1725 }
1726
1727 if (ret == RAM_SAVE_CONTROL_DELAYED) {
1728 return true;
1729 }
1730
1731 if (bytes_xmit > 0) {
1732 ram_counters.normal++;
1733 } else if (bytes_xmit == 0) {
1734 ram_counters.duplicate++;
1735 }
1736
1737 return true;
1738}
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751static int save_normal_page(RAMState *rs, RAMBlock *block, ram_addr_t offset,
1752 uint8_t *buf, bool async)
1753{
1754 ram_counters.transferred += save_page_header(rs, rs->f, block,
1755 offset | RAM_SAVE_FLAG_PAGE);
1756 if (async) {
1757 qemu_put_buffer_async(rs->f, buf, TARGET_PAGE_SIZE,
1758 migrate_release_ram() &
1759 migration_in_postcopy());
1760 } else {
1761 qemu_put_buffer(rs->f, buf, TARGET_PAGE_SIZE);
1762 }
1763 ram_counters.transferred += TARGET_PAGE_SIZE;
1764 ram_counters.normal++;
1765 return 1;
1766}
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
1782{
1783 int pages = -1;
1784 uint8_t *p;
1785 bool send_async = true;
1786 RAMBlock *block = pss->block;
1787 ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
1788 ram_addr_t current_addr = block->offset + offset;
1789
1790 p = block->host + offset;
1791 trace_ram_save_page(block->idstr, (uint64_t)offset, p);
1792
1793 XBZRLE_cache_lock();
1794 if (!rs->ram_bulk_stage && !migration_in_postcopy() &&
1795 migrate_use_xbzrle()) {
1796 pages = save_xbzrle_page(rs, &p, current_addr, block,
1797 offset, last_stage);
1798 if (!last_stage) {
1799
1800
1801
1802 send_async = false;
1803 }
1804 }
1805
1806
1807 if (pages == -1) {
1808 pages = save_normal_page(rs, block, offset, p, send_async);
1809 }
1810
1811 XBZRLE_cache_unlock();
1812
1813 return pages;
1814}
1815
1816static int ram_save_multifd_page(RAMState *rs, RAMBlock *block,
1817 ram_addr_t offset)
1818{
1819 multifd_queue_page(block, offset);
1820 ram_counters.normal++;
1821
1822 return 1;
1823}
1824
1825static int do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
1826 ram_addr_t offset, uint8_t *source_buf)
1827{
1828 RAMState *rs = ram_state;
1829 int bytes_sent, blen;
1830 uint8_t *p = block->host + (offset & TARGET_PAGE_MASK);
1831
1832 bytes_sent = save_page_header(rs, f, block, offset |
1833 RAM_SAVE_FLAG_COMPRESS_PAGE);
1834
1835
1836
1837
1838
1839
1840 memcpy(source_buf, p, TARGET_PAGE_SIZE);
1841 blen = qemu_put_compression_data(f, stream, source_buf, TARGET_PAGE_SIZE);
1842 if (blen < 0) {
1843 bytes_sent = 0;
1844 qemu_file_set_error(migrate_get_current()->to_dst_file, blen);
1845 error_report("compressed data failed!");
1846 } else {
1847 bytes_sent += blen;
1848 ram_release_pages(block->idstr, offset & TARGET_PAGE_MASK, 1);
1849 }
1850
1851 return bytes_sent;
1852}
1853
1854static void flush_compressed_data(RAMState *rs)
1855{
1856 int idx, len, thread_count;
1857
1858 if (!migrate_use_compression()) {
1859 return;
1860 }
1861 thread_count = migrate_compress_threads();
1862
1863 qemu_mutex_lock(&comp_done_lock);
1864 for (idx = 0; idx < thread_count; idx++) {
1865 while (!comp_param[idx].done) {
1866 qemu_cond_wait(&comp_done_cond, &comp_done_lock);
1867 }
1868 }
1869 qemu_mutex_unlock(&comp_done_lock);
1870
1871 for (idx = 0; idx < thread_count; idx++) {
1872 qemu_mutex_lock(&comp_param[idx].mutex);
1873 if (!comp_param[idx].quit) {
1874 len = qemu_put_qemu_file(rs->f, comp_param[idx].file);
1875 ram_counters.transferred += len;
1876 }
1877 qemu_mutex_unlock(&comp_param[idx].mutex);
1878 }
1879}
1880
1881static inline void set_compress_params(CompressParam *param, RAMBlock *block,
1882 ram_addr_t offset)
1883{
1884 param->block = block;
1885 param->offset = offset;
1886}
1887
1888static int compress_page_with_multi_thread(RAMState *rs, RAMBlock *block,
1889 ram_addr_t offset)
1890{
1891 int idx, thread_count, bytes_xmit = -1, pages = -1;
1892
1893 thread_count = migrate_compress_threads();
1894 qemu_mutex_lock(&comp_done_lock);
1895 while (true) {
1896 for (idx = 0; idx < thread_count; idx++) {
1897 if (comp_param[idx].done) {
1898 comp_param[idx].done = false;
1899 bytes_xmit = qemu_put_qemu_file(rs->f, comp_param[idx].file);
1900 qemu_mutex_lock(&comp_param[idx].mutex);
1901 set_compress_params(&comp_param[idx], block, offset);
1902 qemu_cond_signal(&comp_param[idx].cond);
1903 qemu_mutex_unlock(&comp_param[idx].mutex);
1904 pages = 1;
1905 ram_counters.normal++;
1906 ram_counters.transferred += bytes_xmit;
1907 break;
1908 }
1909 }
1910 if (pages > 0) {
1911 break;
1912 } else {
1913 qemu_cond_wait(&comp_done_cond, &comp_done_lock);
1914 }
1915 }
1916 qemu_mutex_unlock(&comp_done_lock);
1917
1918 return pages;
1919}
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931static bool find_dirty_block(RAMState *rs, PageSearchStatus *pss, bool *again)
1932{
1933 pss->page = migration_bitmap_find_dirty(rs, pss->block, pss->page);
1934 if (pss->complete_round && pss->block == rs->last_seen_block &&
1935 pss->page >= rs->last_page) {
1936
1937
1938
1939
1940 *again = false;
1941 return false;
1942 }
1943 if ((pss->page << TARGET_PAGE_BITS) >= pss->block->used_length) {
1944
1945 pss->page = 0;
1946 pss->block = QLIST_NEXT_RCU(pss->block, next);
1947 if (!pss->block) {
1948
1949 pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1950
1951 pss->complete_round = true;
1952 rs->ram_bulk_stage = false;
1953 if (migrate_use_xbzrle()) {
1954
1955
1956
1957 flush_compressed_data(rs);
1958 }
1959 }
1960
1961 *again = true;
1962 return false;
1963 } else {
1964
1965 *again = true;
1966
1967 return true;
1968 }
1969}
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
1982{
1983 RAMBlock *block = NULL;
1984
1985 qemu_mutex_lock(&rs->src_page_req_mutex);
1986 if (!QSIMPLEQ_EMPTY(&rs->src_page_requests)) {
1987 struct RAMSrcPageRequest *entry =
1988 QSIMPLEQ_FIRST(&rs->src_page_requests);
1989 block = entry->rb;
1990 *offset = entry->offset;
1991
1992 if (entry->len > TARGET_PAGE_SIZE) {
1993 entry->len -= TARGET_PAGE_SIZE;
1994 entry->offset += TARGET_PAGE_SIZE;
1995 } else {
1996 memory_region_unref(block->mr);
1997 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1998 g_free(entry);
1999 migration_consume_urgent_request();
2000 }
2001 }
2002 qemu_mutex_unlock(&rs->src_page_req_mutex);
2003
2004 return block;
2005}
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
2018{
2019 RAMBlock *block;
2020 ram_addr_t offset;
2021 bool dirty;
2022
2023 do {
2024 block = unqueue_page(rs, &offset);
2025
2026
2027
2028
2029
2030
2031 if (block) {
2032 unsigned long page;
2033
2034 page = offset >> TARGET_PAGE_BITS;
2035 dirty = test_bit(page, block->bmap);
2036 if (!dirty) {
2037 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
2038 page, test_bit(page, block->unsentmap));
2039 } else {
2040 trace_get_queued_page(block->idstr, (uint64_t)offset, page);
2041 }
2042 }
2043
2044 } while (block && !dirty);
2045
2046 if (block) {
2047
2048
2049
2050
2051
2052
2053 rs->ram_bulk_stage = false;
2054
2055
2056
2057
2058
2059
2060 pss->block = block;
2061 pss->page = offset >> TARGET_PAGE_BITS;
2062 }
2063
2064 return !!block;
2065}
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075static void migration_page_queue_free(RAMState *rs)
2076{
2077 struct RAMSrcPageRequest *mspr, *next_mspr;
2078
2079
2080
2081 rcu_read_lock();
2082 QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
2083 memory_region_unref(mspr->rb->mr);
2084 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
2085 g_free(mspr);
2086 }
2087 rcu_read_unlock();
2088}
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len)
2103{
2104 RAMBlock *ramblock;
2105 RAMState *rs = ram_state;
2106
2107 ram_counters.postcopy_requests++;
2108 rcu_read_lock();
2109 if (!rbname) {
2110
2111 ramblock = rs->last_req_rb;
2112
2113 if (!ramblock) {
2114
2115
2116
2117
2118 error_report("ram_save_queue_pages no previous block");
2119 goto err;
2120 }
2121 } else {
2122 ramblock = qemu_ram_block_by_name(rbname);
2123
2124 if (!ramblock) {
2125
2126 error_report("ram_save_queue_pages no block '%s'", rbname);
2127 goto err;
2128 }
2129 rs->last_req_rb = ramblock;
2130 }
2131 trace_ram_save_queue_pages(ramblock->idstr, start, len);
2132 if (start+len > ramblock->used_length) {
2133 error_report("%s request overrun start=" RAM_ADDR_FMT " len="
2134 RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
2135 __func__, start, len, ramblock->used_length);
2136 goto err;
2137 }
2138
2139 struct RAMSrcPageRequest *new_entry =
2140 g_malloc0(sizeof(struct RAMSrcPageRequest));
2141 new_entry->rb = ramblock;
2142 new_entry->offset = start;
2143 new_entry->len = len;
2144
2145 memory_region_ref(ramblock->mr);
2146 qemu_mutex_lock(&rs->src_page_req_mutex);
2147 QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
2148 migration_make_urgent_request();
2149 qemu_mutex_unlock(&rs->src_page_req_mutex);
2150 rcu_read_unlock();
2151
2152 return 0;
2153
2154err:
2155 rcu_read_unlock();
2156 return -1;
2157}
2158
2159static bool save_page_use_compression(RAMState *rs)
2160{
2161 if (!migrate_use_compression()) {
2162 return false;
2163 }
2164
2165
2166
2167
2168
2169
2170 if (rs->ram_bulk_stage || !migrate_use_xbzrle()) {
2171 return true;
2172 }
2173
2174 return false;
2175}
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186static int ram_save_target_page(RAMState *rs, PageSearchStatus *pss,
2187 bool last_stage)
2188{
2189 RAMBlock *block = pss->block;
2190 ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
2191 int res;
2192
2193 if (control_save_page(rs, block, offset, &res)) {
2194 return res;
2195 }
2196
2197
2198
2199
2200
2201
2202
2203
2204 if (block != rs->last_sent_block && save_page_use_compression(rs)) {
2205 flush_compressed_data(rs);
2206 }
2207
2208 res = save_zero_page(rs, block, offset);
2209 if (res > 0) {
2210
2211
2212
2213 if (!save_page_use_compression(rs)) {
2214 XBZRLE_cache_lock();
2215 xbzrle_cache_zero_page(rs, block->offset + offset);
2216 XBZRLE_cache_unlock();
2217 }
2218 ram_release_pages(block->idstr, offset, res);
2219 return res;
2220 }
2221
2222
2223
2224
2225
2226
2227
2228 if (block == rs->last_sent_block && save_page_use_compression(rs)) {
2229 return compress_page_with_multi_thread(rs, block, offset);
2230 } else if (migrate_use_multifd()) {
2231 return ram_save_multifd_page(rs, block, offset);
2232 }
2233
2234 return ram_save_page(rs, pss, last_stage);
2235}
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss,
2256 bool last_stage)
2257{
2258 int tmppages, pages = 0;
2259 size_t pagesize_bits =
2260 qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2261
2262 if (!qemu_ram_is_migratable(pss->block)) {
2263 error_report("block %s should not be migrated !", pss->block->idstr);
2264 return 0;
2265 }
2266
2267 do {
2268
2269 if (!migration_bitmap_clear_dirty(rs, pss->block, pss->page)) {
2270 pss->page++;
2271 continue;
2272 }
2273
2274 tmppages = ram_save_target_page(rs, pss, last_stage);
2275 if (tmppages < 0) {
2276 return tmppages;
2277 }
2278
2279 pages += tmppages;
2280 if (pss->block->unsentmap) {
2281 clear_bit(pss->page, pss->block->unsentmap);
2282 }
2283
2284 pss->page++;
2285 } while ((pss->page & (pagesize_bits - 1)) &&
2286 offset_in_ramblock(pss->block, pss->page << TARGET_PAGE_BITS));
2287
2288
2289 pss->page--;
2290 return pages;
2291}
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307static int ram_find_and_save_block(RAMState *rs, bool last_stage)
2308{
2309 PageSearchStatus pss;
2310 int pages = 0;
2311 bool again, found;
2312
2313
2314 if (!ram_bytes_total()) {
2315 return pages;
2316 }
2317
2318 pss.block = rs->last_seen_block;
2319 pss.page = rs->last_page;
2320 pss.complete_round = false;
2321
2322 if (!pss.block) {
2323 pss.block = QLIST_FIRST_RCU(&ram_list.blocks);
2324 }
2325
2326 do {
2327 again = true;
2328 found = get_queued_page(rs, &pss);
2329
2330 if (!found) {
2331
2332 found = find_dirty_block(rs, &pss, &again);
2333 }
2334
2335 if (found) {
2336 pages = ram_save_host_page(rs, &pss, last_stage);
2337 }
2338 } while (!pages && again);
2339
2340 rs->last_seen_block = pss.block;
2341 rs->last_page = pss.page;
2342
2343 return pages;
2344}
2345
2346void acct_update_position(QEMUFile *f, size_t size, bool zero)
2347{
2348 uint64_t pages = size / TARGET_PAGE_SIZE;
2349
2350 if (zero) {
2351 ram_counters.duplicate += pages;
2352 } else {
2353 ram_counters.normal += pages;
2354 ram_counters.transferred += size;
2355 qemu_update_position(f, size);
2356 }
2357}
2358
2359uint64_t ram_bytes_total(void)
2360{
2361 RAMBlock *block;
2362 uint64_t total = 0;
2363
2364 rcu_read_lock();
2365 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2366 total += block->used_length;
2367 }
2368 rcu_read_unlock();
2369 return total;
2370}
2371
2372static void xbzrle_load_setup(void)
2373{
2374 XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
2375}
2376
2377static void xbzrle_load_cleanup(void)
2378{
2379 g_free(XBZRLE.decoded_buf);
2380 XBZRLE.decoded_buf = NULL;
2381}
2382
2383static void ram_state_cleanup(RAMState **rsp)
2384{
2385 if (*rsp) {
2386 migration_page_queue_free(*rsp);
2387 qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
2388 qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
2389 g_free(*rsp);
2390 *rsp = NULL;
2391 }
2392}
2393
2394static void xbzrle_cleanup(void)
2395{
2396 XBZRLE_cache_lock();
2397 if (XBZRLE.cache) {
2398 cache_fini(XBZRLE.cache);
2399 g_free(XBZRLE.encoded_buf);
2400 g_free(XBZRLE.current_buf);
2401 g_free(XBZRLE.zero_target_page);
2402 XBZRLE.cache = NULL;
2403 XBZRLE.encoded_buf = NULL;
2404 XBZRLE.current_buf = NULL;
2405 XBZRLE.zero_target_page = NULL;
2406 }
2407 XBZRLE_cache_unlock();
2408}
2409
2410static void ram_save_cleanup(void *opaque)
2411{
2412 RAMState **rsp = opaque;
2413 RAMBlock *block;
2414
2415
2416
2417
2418 memory_global_dirty_log_stop();
2419
2420 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2421 g_free(block->bmap);
2422 block->bmap = NULL;
2423 g_free(block->unsentmap);
2424 block->unsentmap = NULL;
2425 }
2426
2427 xbzrle_cleanup();
2428 compress_threads_save_cleanup();
2429 ram_state_cleanup(rsp);
2430}
2431
2432static void ram_state_reset(RAMState *rs)
2433{
2434 rs->last_seen_block = NULL;
2435 rs->last_sent_block = NULL;
2436 rs->last_page = 0;
2437 rs->last_version = ram_list.version;
2438 rs->ram_bulk_stage = true;
2439}
2440
2441#define MAX_WAIT 50
2442
2443
2444
2445
2446
2447
2448void ram_debug_dump_bitmap(unsigned long *todump, bool expected,
2449 unsigned long pages)
2450{
2451 int64_t cur;
2452 int64_t linelen = 128;
2453 char linebuf[129];
2454
2455 for (cur = 0; cur < pages; cur += linelen) {
2456 int64_t curb;
2457 bool found = false;
2458
2459
2460
2461
2462 if (cur + linelen > pages) {
2463 linelen = pages - cur;
2464 }
2465 for (curb = 0; curb < linelen; curb++) {
2466 bool thisbit = test_bit(cur + curb, todump);
2467 linebuf[curb] = thisbit ? '1' : '.';
2468 found = found || (thisbit != expected);
2469 }
2470 if (found) {
2471 linebuf[curb] = '\0';
2472 fprintf(stderr, "0x%08" PRIx64 " : %s\n", cur, linebuf);
2473 }
2474 }
2475}
2476
2477
2478
2479void ram_postcopy_migrated_memory_release(MigrationState *ms)
2480{
2481 struct RAMBlock *block;
2482
2483 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2484 unsigned long *bitmap = block->bmap;
2485 unsigned long range = block->used_length >> TARGET_PAGE_BITS;
2486 unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
2487
2488 while (run_start < range) {
2489 unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
2490 ram_discard_range(block->idstr, run_start << TARGET_PAGE_BITS,
2491 (run_end - run_start) << TARGET_PAGE_BITS);
2492 run_start = find_next_zero_bit(bitmap, range, run_end + 1);
2493 }
2494 }
2495}
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511static int postcopy_send_discard_bm_ram(MigrationState *ms,
2512 PostcopyDiscardState *pds,
2513 RAMBlock *block)
2514{
2515 unsigned long end = block->used_length >> TARGET_PAGE_BITS;
2516 unsigned long current;
2517 unsigned long *unsentmap = block->unsentmap;
2518
2519 for (current = 0; current < end; ) {
2520 unsigned long one = find_next_bit(unsentmap, end, current);
2521
2522 if (one <= end) {
2523 unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1);
2524 unsigned long discard_length;
2525
2526 if (zero >= end) {
2527 discard_length = end - one;
2528 } else {
2529 discard_length = zero - one;
2530 }
2531 if (discard_length) {
2532 postcopy_discard_send_range(ms, pds, one, discard_length);
2533 }
2534 current = one + discard_length;
2535 } else {
2536 current = one;
2537 }
2538 }
2539
2540 return 0;
2541}
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556static int postcopy_each_ram_send_discard(MigrationState *ms)
2557{
2558 struct RAMBlock *block;
2559 int ret;
2560
2561 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2562 PostcopyDiscardState *pds =
2563 postcopy_discard_send_init(ms, block->idstr);
2564
2565
2566
2567
2568
2569
2570 ret = postcopy_send_discard_bm_ram(ms, pds, block);
2571 postcopy_discard_send_finish(ms, pds);
2572 if (ret) {
2573 return ret;
2574 }
2575 }
2576
2577 return 0;
2578}
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass,
2597 RAMBlock *block,
2598 PostcopyDiscardState *pds)
2599{
2600 RAMState *rs = ram_state;
2601 unsigned long *bitmap = block->bmap;
2602 unsigned long *unsentmap = block->unsentmap;
2603 unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
2604 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
2605 unsigned long run_start;
2606
2607 if (block->page_size == TARGET_PAGE_SIZE) {
2608
2609 return;
2610 }
2611
2612 if (unsent_pass) {
2613
2614 run_start = find_next_zero_bit(unsentmap, pages, 0);
2615 } else {
2616
2617 run_start = find_next_bit(bitmap, pages, 0);
2618 }
2619
2620 while (run_start < pages) {
2621 bool do_fixup = false;
2622 unsigned long fixup_start_addr;
2623 unsigned long host_offset;
2624
2625
2626
2627
2628
2629 host_offset = run_start % host_ratio;
2630 if (host_offset) {
2631 do_fixup = true;
2632 run_start -= host_offset;
2633 fixup_start_addr = run_start;
2634
2635 run_start = run_start + host_ratio;
2636 } else {
2637
2638 unsigned long run_end;
2639 if (unsent_pass) {
2640 run_end = find_next_bit(unsentmap, pages, run_start + 1);
2641 } else {
2642 run_end = find_next_zero_bit(bitmap, pages, run_start + 1);
2643 }
2644
2645
2646
2647
2648
2649 host_offset = run_end % host_ratio;
2650 if (host_offset) {
2651 do_fixup = true;
2652 fixup_start_addr = run_end - host_offset;
2653
2654
2655
2656
2657 run_start = fixup_start_addr + host_ratio;
2658 } else {
2659
2660
2661
2662
2663 run_start = run_end + 1;
2664 }
2665 }
2666
2667 if (do_fixup) {
2668 unsigned long page;
2669
2670
2671 if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) {
2672
2673
2674
2675
2676
2677
2678
2679 postcopy_discard_send_range(ms, pds, fixup_start_addr,
2680 host_ratio);
2681 }
2682
2683
2684 for (page = fixup_start_addr;
2685 page < fixup_start_addr + host_ratio; page++) {
2686
2687 set_bit(page, unsentmap);
2688
2689
2690
2691
2692
2693 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
2694 }
2695 }
2696
2697 if (unsent_pass) {
2698
2699 run_start = find_next_zero_bit(unsentmap, pages, run_start);
2700 } else {
2701
2702 run_start = find_next_bit(bitmap, pages, run_start);
2703 }
2704 }
2705}
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721static int postcopy_chunk_hostpages(MigrationState *ms, RAMBlock *block)
2722{
2723 PostcopyDiscardState *pds =
2724 postcopy_discard_send_init(ms, block->idstr);
2725
2726
2727 postcopy_chunk_hostpages_pass(ms, true, block, pds);
2728
2729
2730
2731
2732 postcopy_chunk_hostpages_pass(ms, false, block, pds);
2733
2734 postcopy_discard_send_finish(ms, pds);
2735 return 0;
2736}
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753int ram_postcopy_send_discard_bitmap(MigrationState *ms)
2754{
2755 RAMState *rs = ram_state;
2756 RAMBlock *block;
2757 int ret;
2758
2759 rcu_read_lock();
2760
2761
2762 migration_bitmap_sync(rs);
2763
2764
2765 rs->last_seen_block = NULL;
2766 rs->last_sent_block = NULL;
2767 rs->last_page = 0;
2768
2769 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2770 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
2771 unsigned long *bitmap = block->bmap;
2772 unsigned long *unsentmap = block->unsentmap;
2773
2774 if (!unsentmap) {
2775
2776
2777
2778
2779 error_report("migration ram resized during precopy phase");
2780 rcu_read_unlock();
2781 return -EINVAL;
2782 }
2783
2784 ret = postcopy_chunk_hostpages(ms, block);
2785 if (ret) {
2786 rcu_read_unlock();
2787 return ret;
2788 }
2789
2790
2791
2792
2793 bitmap_or(unsentmap, unsentmap, bitmap, pages);
2794#ifdef DEBUG_POSTCOPY
2795 ram_debug_dump_bitmap(unsentmap, true, pages);
2796#endif
2797 }
2798 trace_ram_postcopy_send_discard_bitmap();
2799
2800 ret = postcopy_each_ram_send_discard(ms);
2801 rcu_read_unlock();
2802
2803 return ret;
2804}
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816int ram_discard_range(const char *rbname, uint64_t start, size_t length)
2817{
2818 int ret = -1;
2819
2820 trace_ram_discard_range(rbname, start, length);
2821
2822 rcu_read_lock();
2823 RAMBlock *rb = qemu_ram_block_by_name(rbname);
2824
2825 if (!rb) {
2826 error_report("ram_discard_range: Failed to find block '%s'", rbname);
2827 goto err;
2828 }
2829
2830
2831
2832
2833
2834 if (rb->receivedmap) {
2835 bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2836 length >> qemu_target_page_bits());
2837 }
2838
2839 ret = ram_block_discard_range(rb, start, length);
2840
2841err:
2842 rcu_read_unlock();
2843
2844 return ret;
2845}
2846
2847
2848
2849
2850
2851static int xbzrle_init(void)
2852{
2853 Error *local_err = NULL;
2854
2855 if (!migrate_use_xbzrle()) {
2856 return 0;
2857 }
2858
2859 XBZRLE_cache_lock();
2860
2861 XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2862 if (!XBZRLE.zero_target_page) {
2863 error_report("%s: Error allocating zero page", __func__);
2864 goto err_out;
2865 }
2866
2867 XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
2868 TARGET_PAGE_SIZE, &local_err);
2869 if (!XBZRLE.cache) {
2870 error_report_err(local_err);
2871 goto free_zero_page;
2872 }
2873
2874 XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
2875 if (!XBZRLE.encoded_buf) {
2876 error_report("%s: Error allocating encoded_buf", __func__);
2877 goto free_cache;
2878 }
2879
2880 XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
2881 if (!XBZRLE.current_buf) {
2882 error_report("%s: Error allocating current_buf", __func__);
2883 goto free_encoded_buf;
2884 }
2885
2886
2887 XBZRLE_cache_unlock();
2888 return 0;
2889
2890free_encoded_buf:
2891 g_free(XBZRLE.encoded_buf);
2892 XBZRLE.encoded_buf = NULL;
2893free_cache:
2894 cache_fini(XBZRLE.cache);
2895 XBZRLE.cache = NULL;
2896free_zero_page:
2897 g_free(XBZRLE.zero_target_page);
2898 XBZRLE.zero_target_page = NULL;
2899err_out:
2900 XBZRLE_cache_unlock();
2901 return -ENOMEM;
2902}
2903
2904static int ram_state_init(RAMState **rsp)
2905{
2906 *rsp = g_try_new0(RAMState, 1);
2907
2908 if (!*rsp) {
2909 error_report("%s: Init ramstate fail", __func__);
2910 return -1;
2911 }
2912
2913 qemu_mutex_init(&(*rsp)->bitmap_mutex);
2914 qemu_mutex_init(&(*rsp)->src_page_req_mutex);
2915 QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
2916
2917
2918
2919
2920
2921 (*rsp)->migration_dirty_pages = ram_bytes_total() >> TARGET_PAGE_BITS;
2922
2923 ram_state_reset(*rsp);
2924
2925 return 0;
2926}
2927
2928static void ram_list_init_bitmaps(void)
2929{
2930 RAMBlock *block;
2931 unsigned long pages;
2932
2933
2934 if (ram_bytes_total()) {
2935 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2936 pages = block->max_length >> TARGET_PAGE_BITS;
2937 block->bmap = bitmap_new(pages);
2938 bitmap_set(block->bmap, 0, pages);
2939 if (migrate_postcopy_ram()) {
2940 block->unsentmap = bitmap_new(pages);
2941 bitmap_set(block->unsentmap, 0, pages);
2942 }
2943 }
2944 }
2945}
2946
2947static void ram_init_bitmaps(RAMState *rs)
2948{
2949
2950 qemu_mutex_lock_iothread();
2951 qemu_mutex_lock_ramlist();
2952 rcu_read_lock();
2953
2954 ram_list_init_bitmaps();
2955 memory_global_dirty_log_start();
2956 migration_bitmap_sync(rs);
2957
2958 rcu_read_unlock();
2959 qemu_mutex_unlock_ramlist();
2960 qemu_mutex_unlock_iothread();
2961}
2962
2963static int ram_init_all(RAMState **rsp)
2964{
2965 if (ram_state_init(rsp)) {
2966 return -1;
2967 }
2968
2969 if (xbzrle_init()) {
2970 ram_state_cleanup(rsp);
2971 return -1;
2972 }
2973
2974 ram_init_bitmaps(*rsp);
2975
2976 return 0;
2977}
2978
2979static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
2980{
2981 RAMBlock *block;
2982 uint64_t pages = 0;
2983
2984
2985
2986
2987
2988
2989
2990 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2991 pages += bitmap_count_one(block->bmap,
2992 block->used_length >> TARGET_PAGE_BITS);
2993 }
2994
2995
2996 rs->migration_dirty_pages = pages;
2997
2998 rs->last_seen_block = NULL;
2999 rs->last_sent_block = NULL;
3000 rs->last_page = 0;
3001 rs->last_version = ram_list.version;
3002
3003
3004
3005
3006 rs->ram_bulk_stage = false;
3007
3008
3009 rs->f = out;
3010
3011 trace_ram_state_resume_prepare(pages);
3012}
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029static int ram_save_setup(QEMUFile *f, void *opaque)
3030{
3031 RAMState **rsp = opaque;
3032 RAMBlock *block;
3033
3034 if (compress_threads_save_setup()) {
3035 return -1;
3036 }
3037
3038
3039 if (!migration_in_colo_state()) {
3040 if (ram_init_all(rsp) != 0) {
3041 compress_threads_save_cleanup();
3042 return -1;
3043 }
3044 }
3045 (*rsp)->f = f;
3046
3047 rcu_read_lock();
3048
3049 qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
3050
3051 RAMBLOCK_FOREACH_MIGRATABLE(block) {
3052 qemu_put_byte(f, strlen(block->idstr));
3053 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
3054 qemu_put_be64(f, block->used_length);
3055 if (migrate_postcopy_ram() && block->page_size != qemu_host_page_size) {
3056 qemu_put_be64(f, block->page_size);
3057 }
3058 }
3059
3060 rcu_read_unlock();
3061
3062 ram_control_before_iterate(f, RAM_CONTROL_SETUP);
3063 ram_control_after_iterate(f, RAM_CONTROL_SETUP);
3064
3065 multifd_send_sync_main();
3066 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3067 qemu_fflush(f);
3068
3069 return 0;
3070}
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080static int ram_save_iterate(QEMUFile *f, void *opaque)
3081{
3082 RAMState **temp = opaque;
3083 RAMState *rs = *temp;
3084 int ret;
3085 int i;
3086 int64_t t0;
3087 int done = 0;
3088
3089 if (blk_mig_bulk_active()) {
3090
3091
3092
3093 goto out;
3094 }
3095
3096 rcu_read_lock();
3097 if (ram_list.version != rs->last_version) {
3098 ram_state_reset(rs);
3099 }
3100
3101
3102 smp_rmb();
3103
3104 ram_control_before_iterate(f, RAM_CONTROL_ROUND);
3105
3106 t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
3107 i = 0;
3108 while ((ret = qemu_file_rate_limit(f)) == 0 ||
3109 !QSIMPLEQ_EMPTY(&rs->src_page_requests)) {
3110 int pages;
3111
3112 if (qemu_file_get_error(f)) {
3113 break;
3114 }
3115
3116 pages = ram_find_and_save_block(rs, false);
3117
3118 if (pages == 0) {
3119 done = 1;
3120 break;
3121 }
3122 rs->iterations++;
3123
3124
3125
3126
3127
3128
3129 if ((i & 63) == 0) {
3130 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / 1000000;
3131 if (t1 > MAX_WAIT) {
3132 trace_ram_save_iterate_big_wait(t1, i);
3133 break;
3134 }
3135 }
3136 i++;
3137 }
3138 flush_compressed_data(rs);
3139 rcu_read_unlock();
3140
3141
3142
3143
3144
3145 ram_control_after_iterate(f, RAM_CONTROL_ROUND);
3146
3147 multifd_send_sync_main();
3148out:
3149 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3150 qemu_fflush(f);
3151 ram_counters.transferred += 8;
3152
3153 ret = qemu_file_get_error(f);
3154 if (ret < 0) {
3155 return ret;
3156 }
3157
3158 return done;
3159}
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171static int ram_save_complete(QEMUFile *f, void *opaque)
3172{
3173 RAMState **temp = opaque;
3174 RAMState *rs = *temp;
3175
3176 rcu_read_lock();
3177
3178 if (!migration_in_postcopy()) {
3179 migration_bitmap_sync(rs);
3180 }
3181
3182 ram_control_before_iterate(f, RAM_CONTROL_FINISH);
3183
3184
3185
3186
3187 while (true) {
3188 int pages;
3189
3190 pages = ram_find_and_save_block(rs, !migration_in_colo_state());
3191
3192 if (pages == 0) {
3193 break;
3194 }
3195 }
3196
3197 flush_compressed_data(rs);
3198 ram_control_after_iterate(f, RAM_CONTROL_FINISH);
3199
3200 rcu_read_unlock();
3201
3202 multifd_send_sync_main();
3203 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3204 qemu_fflush(f);
3205
3206 return 0;
3207}
3208
3209static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
3210 uint64_t *res_precopy_only,
3211 uint64_t *res_compatible,
3212 uint64_t *res_postcopy_only)
3213{
3214 RAMState **temp = opaque;
3215 RAMState *rs = *temp;
3216 uint64_t remaining_size;
3217
3218 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3219
3220 if (!migration_in_postcopy() &&
3221 remaining_size < max_size) {
3222 qemu_mutex_lock_iothread();
3223 rcu_read_lock();
3224 migration_bitmap_sync(rs);
3225 rcu_read_unlock();
3226 qemu_mutex_unlock_iothread();
3227 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3228 }
3229
3230 if (migrate_postcopy_ram()) {
3231
3232 *res_compatible += remaining_size;
3233 } else {
3234 *res_precopy_only += remaining_size;
3235 }
3236}
3237
3238static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
3239{
3240 unsigned int xh_len;
3241 int xh_flags;
3242 uint8_t *loaded_data;
3243
3244
3245 xh_flags = qemu_get_byte(f);
3246 xh_len = qemu_get_be16(f);
3247
3248 if (xh_flags != ENCODING_FLAG_XBZRLE) {
3249 error_report("Failed to load XBZRLE page - wrong compression!");
3250 return -1;
3251 }
3252
3253 if (xh_len > TARGET_PAGE_SIZE) {
3254 error_report("Failed to load XBZRLE page - len overflow!");
3255 return -1;
3256 }
3257 loaded_data = XBZRLE.decoded_buf;
3258
3259
3260 qemu_get_buffer_in_place(f, &loaded_data, xh_len);
3261
3262
3263 if (xbzrle_decode_buffer(loaded_data, xh_len, host,
3264 TARGET_PAGE_SIZE) == -1) {
3265 error_report("Failed to load XBZRLE page - decode error!");
3266 return -1;
3267 }
3268
3269 return 0;
3270}
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282static inline RAMBlock *ram_block_from_stream(QEMUFile *f, int flags)
3283{
3284 static RAMBlock *block = NULL;
3285 char id[256];
3286 uint8_t len;
3287
3288 if (flags & RAM_SAVE_FLAG_CONTINUE) {
3289 if (!block) {
3290 error_report("Ack, bad migration stream!");
3291 return NULL;
3292 }
3293 return block;
3294 }
3295
3296 len = qemu_get_byte(f);
3297 qemu_get_buffer(f, (uint8_t *)id, len);
3298 id[len] = 0;
3299
3300 block = qemu_ram_block_by_name(id);
3301 if (!block) {
3302 error_report("Can't find block %s", id);
3303 return NULL;
3304 }
3305
3306 if (!qemu_ram_is_migratable(block)) {
3307 error_report("block %s should not be migrated !", id);
3308 return NULL;
3309 }
3310
3311 return block;
3312}
3313
3314static inline void *host_from_ram_block_offset(RAMBlock *block,
3315 ram_addr_t offset)
3316{
3317 if (!offset_in_ramblock(block, offset)) {
3318 return NULL;
3319 }
3320
3321 return block->host + offset;
3322}
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
3335{
3336 if (ch != 0 || !is_zero_range(host, size)) {
3337 memset(host, ch, size);
3338 }
3339}
3340
3341
3342static int
3343qemu_uncompress_data(z_stream *stream, uint8_t *dest, size_t dest_len,
3344 const uint8_t *source, size_t source_len)
3345{
3346 int err;
3347
3348 err = inflateReset(stream);
3349 if (err != Z_OK) {
3350 return -1;
3351 }
3352
3353 stream->avail_in = source_len;
3354 stream->next_in = (uint8_t *)source;
3355 stream->avail_out = dest_len;
3356 stream->next_out = dest;
3357
3358 err = inflate(stream, Z_NO_FLUSH);
3359 if (err != Z_STREAM_END) {
3360 return -1;
3361 }
3362
3363 return stream->total_out;
3364}
3365
3366static void *do_data_decompress(void *opaque)
3367{
3368 DecompressParam *param = opaque;
3369 unsigned long pagesize;
3370 uint8_t *des;
3371 int len, ret;
3372
3373 qemu_mutex_lock(¶m->mutex);
3374 while (!param->quit) {
3375 if (param->des) {
3376 des = param->des;
3377 len = param->len;
3378 param->des = 0;
3379 qemu_mutex_unlock(¶m->mutex);
3380
3381 pagesize = TARGET_PAGE_SIZE;
3382
3383 ret = qemu_uncompress_data(¶m->stream, des, pagesize,
3384 param->compbuf, len);
3385 if (ret < 0 && migrate_get_current()->decompress_error_check) {
3386 error_report("decompress data failed");
3387 qemu_file_set_error(decomp_file, ret);
3388 }
3389
3390 qemu_mutex_lock(&decomp_done_lock);
3391 param->done = true;
3392 qemu_cond_signal(&decomp_done_cond);
3393 qemu_mutex_unlock(&decomp_done_lock);
3394
3395 qemu_mutex_lock(¶m->mutex);
3396 } else {
3397 qemu_cond_wait(¶m->cond, ¶m->mutex);
3398 }
3399 }
3400 qemu_mutex_unlock(¶m->mutex);
3401
3402 return NULL;
3403}
3404
3405static int wait_for_decompress_done(void)
3406{
3407 int idx, thread_count;
3408
3409 if (!migrate_use_compression()) {
3410 return 0;
3411 }
3412
3413 thread_count = migrate_decompress_threads();
3414 qemu_mutex_lock(&decomp_done_lock);
3415 for (idx = 0; idx < thread_count; idx++) {
3416 while (!decomp_param[idx].done) {
3417 qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
3418 }
3419 }
3420 qemu_mutex_unlock(&decomp_done_lock);
3421 return qemu_file_get_error(decomp_file);
3422}
3423
3424static void compress_threads_load_cleanup(void)
3425{
3426 int i, thread_count;
3427
3428 if (!migrate_use_compression()) {
3429 return;
3430 }
3431 thread_count = migrate_decompress_threads();
3432 for (i = 0; i < thread_count; i++) {
3433
3434
3435
3436
3437 if (!decomp_param[i].compbuf) {
3438 break;
3439 }
3440
3441 qemu_mutex_lock(&decomp_param[i].mutex);
3442 decomp_param[i].quit = true;
3443 qemu_cond_signal(&decomp_param[i].cond);
3444 qemu_mutex_unlock(&decomp_param[i].mutex);
3445 }
3446 for (i = 0; i < thread_count; i++) {
3447 if (!decomp_param[i].compbuf) {
3448 break;
3449 }
3450
3451 qemu_thread_join(decompress_threads + i);
3452 qemu_mutex_destroy(&decomp_param[i].mutex);
3453 qemu_cond_destroy(&decomp_param[i].cond);
3454 inflateEnd(&decomp_param[i].stream);
3455 g_free(decomp_param[i].compbuf);
3456 decomp_param[i].compbuf = NULL;
3457 }
3458 g_free(decompress_threads);
3459 g_free(decomp_param);
3460 decompress_threads = NULL;
3461 decomp_param = NULL;
3462 decomp_file = NULL;
3463}
3464
3465static int compress_threads_load_setup(QEMUFile *f)
3466{
3467 int i, thread_count;
3468
3469 if (!migrate_use_compression()) {
3470 return 0;
3471 }
3472
3473 thread_count = migrate_decompress_threads();
3474 decompress_threads = g_new0(QemuThread, thread_count);
3475 decomp_param = g_new0(DecompressParam, thread_count);
3476 qemu_mutex_init(&decomp_done_lock);
3477 qemu_cond_init(&decomp_done_cond);
3478 decomp_file = f;
3479 for (i = 0; i < thread_count; i++) {
3480 if (inflateInit(&decomp_param[i].stream) != Z_OK) {
3481 goto exit;
3482 }
3483
3484 decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
3485 qemu_mutex_init(&decomp_param[i].mutex);
3486 qemu_cond_init(&decomp_param[i].cond);
3487 decomp_param[i].done = true;
3488 decomp_param[i].quit = false;
3489 qemu_thread_create(decompress_threads + i, "decompress",
3490 do_data_decompress, decomp_param + i,
3491 QEMU_THREAD_JOINABLE);
3492 }
3493 return 0;
3494exit:
3495 compress_threads_load_cleanup();
3496 return -1;
3497}
3498
3499static void decompress_data_with_multi_threads(QEMUFile *f,
3500 void *host, int len)
3501{
3502 int idx, thread_count;
3503
3504 thread_count = migrate_decompress_threads();
3505 qemu_mutex_lock(&decomp_done_lock);
3506 while (true) {
3507 for (idx = 0; idx < thread_count; idx++) {
3508 if (decomp_param[idx].done) {
3509 decomp_param[idx].done = false;
3510 qemu_mutex_lock(&decomp_param[idx].mutex);
3511 qemu_get_buffer(f, decomp_param[idx].compbuf, len);
3512 decomp_param[idx].des = host;
3513 decomp_param[idx].len = len;
3514 qemu_cond_signal(&decomp_param[idx].cond);
3515 qemu_mutex_unlock(&decomp_param[idx].mutex);
3516 break;
3517 }
3518 }
3519 if (idx < thread_count) {
3520 break;
3521 } else {
3522 qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
3523 }
3524 }
3525 qemu_mutex_unlock(&decomp_done_lock);
3526}
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536static int ram_load_setup(QEMUFile *f, void *opaque)
3537{
3538 if (compress_threads_load_setup(f)) {
3539 return -1;
3540 }
3541
3542 xbzrle_load_setup();
3543 ramblock_recv_map_init();
3544 return 0;
3545}
3546
3547static int ram_load_cleanup(void *opaque)
3548{
3549 RAMBlock *rb;
3550 xbzrle_load_cleanup();
3551 compress_threads_load_cleanup();
3552
3553 RAMBLOCK_FOREACH_MIGRATABLE(rb) {
3554 g_free(rb->receivedmap);
3555 rb->receivedmap = NULL;
3556 }
3557 return 0;
3558}
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571int ram_postcopy_incoming_init(MigrationIncomingState *mis)
3572{
3573 return postcopy_ram_incoming_init(mis);
3574}
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586static int ram_load_postcopy(QEMUFile *f)
3587{
3588 int flags = 0, ret = 0;
3589 bool place_needed = false;
3590 bool matches_target_page_size = false;
3591 MigrationIncomingState *mis = migration_incoming_get_current();
3592
3593 void *postcopy_host_page = postcopy_get_tmp_page(mis);
3594 void *last_host = NULL;
3595 bool all_zero = false;
3596
3597 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3598 ram_addr_t addr;
3599 void *host = NULL;
3600 void *page_buffer = NULL;
3601 void *place_source = NULL;
3602 RAMBlock *block = NULL;
3603 uint8_t ch;
3604
3605 addr = qemu_get_be64(f);
3606
3607
3608
3609
3610
3611 ret = qemu_file_get_error(f);
3612 if (ret) {
3613 break;
3614 }
3615
3616 flags = addr & ~TARGET_PAGE_MASK;
3617 addr &= TARGET_PAGE_MASK;
3618
3619 trace_ram_load_postcopy_loop((uint64_t)addr, flags);
3620 place_needed = false;
3621 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE)) {
3622 block = ram_block_from_stream(f, flags);
3623
3624 host = host_from_ram_block_offset(block, addr);
3625 if (!host) {
3626 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3627 ret = -EINVAL;
3628 break;
3629 }
3630 matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641 page_buffer = postcopy_host_page +
3642 ((uintptr_t)host & (block->page_size - 1));
3643
3644 if (!((uintptr_t)host & (block->page_size - 1))) {
3645 all_zero = true;
3646 } else {
3647
3648 if (host != (last_host + TARGET_PAGE_SIZE)) {
3649 error_report("Non-sequential target page %p/%p",
3650 host, last_host);
3651 ret = -EINVAL;
3652 break;
3653 }
3654 }
3655
3656
3657
3658
3659
3660
3661 place_needed = (((uintptr_t)host + TARGET_PAGE_SIZE) &
3662 (block->page_size - 1)) == 0;
3663 place_source = postcopy_host_page;
3664 }
3665 last_host = host;
3666
3667 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
3668 case RAM_SAVE_FLAG_ZERO:
3669 ch = qemu_get_byte(f);
3670 memset(page_buffer, ch, TARGET_PAGE_SIZE);
3671 if (ch) {
3672 all_zero = false;
3673 }
3674 break;
3675
3676 case RAM_SAVE_FLAG_PAGE:
3677 all_zero = false;
3678 if (!matches_target_page_size) {
3679
3680 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
3681 } else {
3682
3683
3684
3685
3686
3687
3688
3689
3690 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
3691 TARGET_PAGE_SIZE);
3692 }
3693 break;
3694 case RAM_SAVE_FLAG_EOS:
3695
3696 multifd_recv_sync_main();
3697 break;
3698 default:
3699 error_report("Unknown combination of migration flags: %#x"
3700 " (postcopy mode)", flags);
3701 ret = -EINVAL;
3702 break;
3703 }
3704
3705
3706 if (!ret && qemu_file_get_error(f)) {
3707 ret = qemu_file_get_error(f);
3708 }
3709
3710 if (!ret && place_needed) {
3711
3712 void *place_dest = host + TARGET_PAGE_SIZE - block->page_size;
3713
3714 if (all_zero) {
3715 ret = postcopy_place_page_zero(mis, place_dest,
3716 block);
3717 } else {
3718 ret = postcopy_place_page(mis, place_dest,
3719 place_source, block);
3720 }
3721 }
3722 }
3723
3724 return ret;
3725}
3726
3727static bool postcopy_is_advised(void)
3728{
3729 PostcopyState ps = postcopy_state_get();
3730 return ps >= POSTCOPY_INCOMING_ADVISE && ps < POSTCOPY_INCOMING_END;
3731}
3732
3733static bool postcopy_is_running(void)
3734{
3735 PostcopyState ps = postcopy_state_get();
3736 return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
3737}
3738
3739static int ram_load(QEMUFile *f, void *opaque, int version_id)
3740{
3741 int flags = 0, ret = 0, invalid_flags = 0;
3742 static uint64_t seq_iter;
3743 int len = 0;
3744
3745
3746
3747
3748 bool postcopy_running = postcopy_is_running();
3749
3750 bool postcopy_advised = postcopy_is_advised();
3751
3752 seq_iter++;
3753
3754 if (version_id != 4) {
3755 ret = -EINVAL;
3756 }
3757
3758 if (!migrate_use_compression()) {
3759 invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE;
3760 }
3761
3762
3763
3764
3765
3766 rcu_read_lock();
3767
3768 if (postcopy_running) {
3769 ret = ram_load_postcopy(f);
3770 }
3771
3772 while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3773 ram_addr_t addr, total_ram_bytes;
3774 void *host = NULL;
3775 uint8_t ch;
3776
3777 addr = qemu_get_be64(f);
3778 flags = addr & ~TARGET_PAGE_MASK;
3779 addr &= TARGET_PAGE_MASK;
3780
3781 if (flags & invalid_flags) {
3782 if (flags & invalid_flags & RAM_SAVE_FLAG_COMPRESS_PAGE) {
3783 error_report("Received an unexpected compressed page");
3784 }
3785
3786 ret = -EINVAL;
3787 break;
3788 }
3789
3790 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
3791 RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
3792 RAMBlock *block = ram_block_from_stream(f, flags);
3793
3794 host = host_from_ram_block_offset(block, addr);
3795 if (!host) {
3796 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3797 ret = -EINVAL;
3798 break;
3799 }
3800 ramblock_recv_bitmap_set(block, host);
3801 trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
3802 }
3803
3804 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
3805 case RAM_SAVE_FLAG_MEM_SIZE:
3806
3807 total_ram_bytes = addr;
3808 while (!ret && total_ram_bytes) {
3809 RAMBlock *block;
3810 char id[256];
3811 ram_addr_t length;
3812
3813 len = qemu_get_byte(f);
3814 qemu_get_buffer(f, (uint8_t *)id, len);
3815 id[len] = 0;
3816 length = qemu_get_be64(f);
3817
3818 block = qemu_ram_block_by_name(id);
3819 if (block && !qemu_ram_is_migratable(block)) {
3820 error_report("block %s should not be migrated !", id);
3821 ret = -EINVAL;
3822 } else if (block) {
3823 if (length != block->used_length) {
3824 Error *local_err = NULL;
3825
3826 ret = qemu_ram_resize(block, length,
3827 &local_err);
3828 if (local_err) {
3829 error_report_err(local_err);
3830 }
3831 }
3832
3833 if (postcopy_advised &&
3834 block->page_size != qemu_host_page_size) {
3835 uint64_t remote_page_size = qemu_get_be64(f);
3836 if (remote_page_size != block->page_size) {
3837 error_report("Mismatched RAM page size %s "
3838 "(local) %zd != %" PRId64,
3839 id, block->page_size,
3840 remote_page_size);
3841 ret = -EINVAL;
3842 }
3843 }
3844 ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
3845 block->idstr);
3846 } else {
3847 error_report("Unknown ramblock \"%s\", cannot "
3848 "accept migration", id);
3849 ret = -EINVAL;
3850 }
3851
3852 total_ram_bytes -= length;
3853 }
3854 break;
3855
3856 case RAM_SAVE_FLAG_ZERO:
3857 ch = qemu_get_byte(f);
3858 ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
3859 break;
3860
3861 case RAM_SAVE_FLAG_PAGE:
3862 qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
3863 break;
3864
3865 case RAM_SAVE_FLAG_COMPRESS_PAGE:
3866 len = qemu_get_be32(f);
3867 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
3868 error_report("Invalid compressed data length: %d", len);
3869 ret = -EINVAL;
3870 break;
3871 }
3872 decompress_data_with_multi_threads(f, host, len);
3873 break;
3874
3875 case RAM_SAVE_FLAG_XBZRLE:
3876 if (load_xbzrle(f, addr, host) < 0) {
3877 error_report("Failed to decompress XBZRLE page at "
3878 RAM_ADDR_FMT, addr);
3879 ret = -EINVAL;
3880 break;
3881 }
3882 break;
3883 case RAM_SAVE_FLAG_EOS:
3884
3885 multifd_recv_sync_main();
3886 break;
3887 default:
3888 if (flags & RAM_SAVE_FLAG_HOOK) {
3889 ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL);
3890 } else {
3891 error_report("Unknown combination of migration flags: %#x",
3892 flags);
3893 ret = -EINVAL;
3894 }
3895 }
3896 if (!ret) {
3897 ret = qemu_file_get_error(f);
3898 }
3899 }
3900
3901 ret |= wait_for_decompress_done();
3902 rcu_read_unlock();
3903 trace_ram_load_complete(ret, seq_iter);
3904 return ret;
3905}
3906
3907static bool ram_has_postcopy(void *opaque)
3908{
3909 return migrate_postcopy_ram();
3910}
3911
3912
3913static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
3914{
3915 RAMBlock *block;
3916 QEMUFile *file = s->to_dst_file;
3917 int ramblock_count = 0;
3918
3919 trace_ram_dirty_bitmap_sync_start();
3920
3921 RAMBLOCK_FOREACH_MIGRATABLE(block) {
3922 qemu_savevm_send_recv_bitmap(file, block->idstr);
3923 trace_ram_dirty_bitmap_request(block->idstr);
3924 ramblock_count++;
3925 }
3926
3927 trace_ram_dirty_bitmap_sync_wait();
3928
3929
3930 while (ramblock_count--) {
3931 qemu_sem_wait(&s->rp_state.rp_sem);
3932 }
3933
3934 trace_ram_dirty_bitmap_sync_complete();
3935
3936 return 0;
3937}
3938
3939static void ram_dirty_bitmap_reload_notify(MigrationState *s)
3940{
3941 qemu_sem_post(&s->rp_state.rp_sem);
3942}
3943
3944
3945
3946
3947
3948
3949int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block)
3950{
3951 int ret = -EINVAL;
3952 QEMUFile *file = s->rp_state.from_dst_file;
3953 unsigned long *le_bitmap, nbits = block->used_length >> TARGET_PAGE_BITS;
3954 uint64_t local_size = DIV_ROUND_UP(nbits, 8);
3955 uint64_t size, end_mark;
3956
3957 trace_ram_dirty_bitmap_reload_begin(block->idstr);
3958
3959 if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
3960 error_report("%s: incorrect state %s", __func__,
3961 MigrationStatus_str(s->state));
3962 return -EINVAL;
3963 }
3964
3965
3966
3967
3968
3969 local_size = ROUND_UP(local_size, 8);
3970
3971
3972 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
3973
3974 size = qemu_get_be64(file);
3975
3976
3977 if (size != local_size) {
3978 error_report("%s: ramblock '%s' bitmap size mismatch "
3979 "(0x%"PRIx64" != 0x%"PRIx64")", __func__,
3980 block->idstr, size, local_size);
3981 ret = -EINVAL;
3982 goto out;
3983 }
3984
3985 size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
3986 end_mark = qemu_get_be64(file);
3987
3988 ret = qemu_file_get_error(file);
3989 if (ret || size != local_size) {
3990 error_report("%s: read bitmap failed for ramblock '%s': %d"
3991 " (size 0x%"PRIx64", got: 0x%"PRIx64")",
3992 __func__, block->idstr, ret, local_size, size);
3993 ret = -EIO;
3994 goto out;
3995 }
3996
3997 if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
3998 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIu64,
3999 __func__, block->idstr, end_mark);
4000 ret = -EINVAL;
4001 goto out;
4002 }
4003
4004
4005
4006
4007
4008 bitmap_from_le(block->bmap, le_bitmap, nbits);
4009
4010
4011
4012
4013
4014 bitmap_complement(block->bmap, block->bmap, nbits);
4015
4016 trace_ram_dirty_bitmap_reload_complete(block->idstr);
4017
4018
4019
4020
4021
4022 ram_dirty_bitmap_reload_notify(s);
4023
4024 ret = 0;
4025out:
4026 g_free(le_bitmap);
4027 return ret;
4028}
4029
4030static int ram_resume_prepare(MigrationState *s, void *opaque)
4031{
4032 RAMState *rs = *(RAMState **)opaque;
4033 int ret;
4034
4035 ret = ram_dirty_bitmap_sync_all(s, rs);
4036 if (ret) {
4037 return ret;
4038 }
4039
4040 ram_state_resume_prepare(rs, s->to_dst_file);
4041
4042 return 0;
4043}
4044
4045static SaveVMHandlers savevm_ram_handlers = {
4046 .save_setup = ram_save_setup,
4047 .save_live_iterate = ram_save_iterate,
4048 .save_live_complete_postcopy = ram_save_complete,
4049 .save_live_complete_precopy = ram_save_complete,
4050 .has_postcopy = ram_has_postcopy,
4051 .save_live_pending = ram_save_pending,
4052 .load_state = ram_load,
4053 .save_cleanup = ram_save_cleanup,
4054 .load_setup = ram_load_setup,
4055 .load_cleanup = ram_load_cleanup,
4056 .resume_prepare = ram_resume_prepare,
4057};
4058
4059void ram_mig_init(void)
4060{
4061 qemu_mutex_init(&XBZRLE.lock);
4062 register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, &ram_state);
4063}
4064