1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#ifndef RAM_ADDR_H
20#define RAM_ADDR_H
21
22#ifndef CONFIG_USER_ONLY
23#include "cpu.h"
24#include "sysemu/xen.h"
25#include "sysemu/tcg.h"
26#include "exec/ramlist.h"
27#include "exec/ramblock.h"
28
29extern uint64_t total_dirty_pages;
30
31
32
33
34
35
36
37
38
39static inline long clear_bmap_size(uint64_t pages, uint8_t shift)
40{
41 return DIV_ROUND_UP(pages, 1UL << shift);
42}
43
44
45
46
47
48
49
50
51
52
53
54static inline void clear_bmap_set(RAMBlock *rb, uint64_t start,
55 uint64_t npages)
56{
57 uint8_t shift = rb->clear_bmap_shift;
58
59 bitmap_set(rb->clear_bmap, start >> shift, clear_bmap_size(npages, shift));
60}
61
62
63
64
65
66
67
68
69
70
71static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page)
72{
73 uint8_t shift = rb->clear_bmap_shift;
74
75 return bitmap_test_and_clear(rb->clear_bmap, page >> shift, 1);
76}
77
78static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
79{
80 return (b && b->host && offset < b->used_length) ? true : false;
81}
82
83static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
84{
85 assert(offset_in_ramblock(block, offset));
86 return (char *)block->host + offset;
87}
88
89static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
90 RAMBlock *rb)
91{
92 uint64_t host_addr_offset =
93 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
94 return host_addr_offset >> TARGET_PAGE_BITS;
95}
96
97bool ramblock_is_pmem(RAMBlock *rb);
98
99long qemu_minrampagesize(void);
100long qemu_maxrampagesize(void);
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
121 uint32_t ram_flags, const char *mem_path,
122 bool readonly, Error **errp);
123RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
124 uint32_t ram_flags, int fd, off_t offset,
125 bool readonly, Error **errp);
126
127RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
128 MemoryRegion *mr, Error **errp);
129RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, MemoryRegion *mr,
130 Error **errp);
131RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
132 void (*resized)(const char*,
133 uint64_t length,
134 void *host),
135 MemoryRegion *mr, Error **errp);
136void qemu_ram_free(RAMBlock *block);
137
138int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
139
140void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length);
141
142
143static inline void qemu_ram_block_writeback(RAMBlock *block)
144{
145 qemu_ram_msync(block, 0, block->used_length);
146}
147
148#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
149#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
150
151static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
152 ram_addr_t length,
153 unsigned client)
154{
155 DirtyMemoryBlocks *blocks;
156 unsigned long end, page;
157 unsigned long idx, offset, base;
158 bool dirty = false;
159
160 assert(client < DIRTY_MEMORY_NUM);
161
162 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
163 page = start >> TARGET_PAGE_BITS;
164
165 WITH_RCU_READ_LOCK_GUARD() {
166 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
167
168 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
169 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
170 base = page - offset;
171 while (page < end) {
172 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
173 unsigned long num = next - base;
174 unsigned long found = find_next_bit(blocks->blocks[idx],
175 num, offset);
176 if (found < num) {
177 dirty = true;
178 break;
179 }
180
181 page = next;
182 idx++;
183 offset = 0;
184 base += DIRTY_MEMORY_BLOCK_SIZE;
185 }
186 }
187
188 return dirty;
189}
190
191static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
192 ram_addr_t length,
193 unsigned client)
194{
195 DirtyMemoryBlocks *blocks;
196 unsigned long end, page;
197 unsigned long idx, offset, base;
198 bool dirty = true;
199
200 assert(client < DIRTY_MEMORY_NUM);
201
202 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
203 page = start >> TARGET_PAGE_BITS;
204
205 RCU_READ_LOCK_GUARD();
206
207 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
208
209 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
210 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
211 base = page - offset;
212 while (page < end) {
213 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
214 unsigned long num = next - base;
215 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
216 if (found < num) {
217 dirty = false;
218 break;
219 }
220
221 page = next;
222 idx++;
223 offset = 0;
224 base += DIRTY_MEMORY_BLOCK_SIZE;
225 }
226
227 return dirty;
228}
229
230static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
231 unsigned client)
232{
233 return cpu_physical_memory_get_dirty(addr, 1, client);
234}
235
236static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
237{
238 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
239 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
240 bool migration =
241 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
242 return !(vga && code && migration);
243}
244
245static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
246 ram_addr_t length,
247 uint8_t mask)
248{
249 uint8_t ret = 0;
250
251 if (mask & (1 << DIRTY_MEMORY_VGA) &&
252 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
253 ret |= (1 << DIRTY_MEMORY_VGA);
254 }
255 if (mask & (1 << DIRTY_MEMORY_CODE) &&
256 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
257 ret |= (1 << DIRTY_MEMORY_CODE);
258 }
259 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
260 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
261 ret |= (1 << DIRTY_MEMORY_MIGRATION);
262 }
263 return ret;
264}
265
266static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
267 unsigned client)
268{
269 unsigned long page, idx, offset;
270 DirtyMemoryBlocks *blocks;
271
272 assert(client < DIRTY_MEMORY_NUM);
273
274 page = addr >> TARGET_PAGE_BITS;
275 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
276 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
277
278 RCU_READ_LOCK_GUARD();
279
280 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
281
282 set_bit_atomic(offset, blocks->blocks[idx]);
283}
284
285static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
286 ram_addr_t length,
287 uint8_t mask)
288{
289 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
290 unsigned long end, page;
291 unsigned long idx, offset, base;
292 int i;
293
294 if (!mask && !xen_enabled()) {
295 return;
296 }
297
298 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
299 page = start >> TARGET_PAGE_BITS;
300
301 WITH_RCU_READ_LOCK_GUARD() {
302 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
303 blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]);
304 }
305
306 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
307 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
308 base = page - offset;
309 while (page < end) {
310 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
311
312 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
313 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
314 offset, next - page);
315 }
316 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
317 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
318 offset, next - page);
319 }
320 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
321 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
322 offset, next - page);
323 }
324
325 page = next;
326 idx++;
327 offset = 0;
328 base += DIRTY_MEMORY_BLOCK_SIZE;
329 }
330 }
331
332 xen_hvm_modified_memory(start, length);
333}
334
335#if !defined(_WIN32)
336static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
337 ram_addr_t start,
338 ram_addr_t pages)
339{
340 unsigned long i, j;
341 unsigned long page_number, c;
342 hwaddr addr;
343 ram_addr_t ram_addr;
344 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
345 unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE;
346 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
347
348
349 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
350 (hpratio == 1)) {
351 unsigned long **blocks[DIRTY_MEMORY_NUM];
352 unsigned long idx;
353 unsigned long offset;
354 long k;
355 long nr = BITS_TO_LONGS(pages);
356
357 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
358 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
359 DIRTY_MEMORY_BLOCK_SIZE);
360
361 WITH_RCU_READ_LOCK_GUARD() {
362 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
363 blocks[i] =
364 qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
365 }
366
367 for (k = 0; k < nr; k++) {
368 if (bitmap[k]) {
369 unsigned long temp = leul_to_cpu(bitmap[k]);
370
371 qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
372
373 if (global_dirty_tracking) {
374 qatomic_or(
375 &blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
376 temp);
377 if (unlikely(
378 global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
379 total_dirty_pages += ctpopl(temp);
380 }
381 }
382
383 if (tcg_enabled()) {
384 qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
385 temp);
386 }
387 }
388
389 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
390 offset = 0;
391 idx++;
392 }
393 }
394 }
395
396 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
397 } else {
398 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
399
400 if (!global_dirty_tracking) {
401 clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
402 }
403
404
405
406
407
408 for (i = 0; i < len; i++) {
409 if (bitmap[i] != 0) {
410 c = leul_to_cpu(bitmap[i]);
411 if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
412 total_dirty_pages += ctpopl(c);
413 }
414 do {
415 j = ctzl(c);
416 c &= ~(1ul << j);
417 page_number = (i * HOST_LONG_BITS + j) * hpratio;
418 addr = page_number * TARGET_PAGE_SIZE;
419 ram_addr = start + addr;
420 cpu_physical_memory_set_dirty_range(ram_addr,
421 TARGET_PAGE_SIZE * hpratio, clients);
422 } while (c != 0);
423 }
424 }
425 }
426}
427#endif
428
429bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
430 ram_addr_t length,
431 unsigned client);
432
433DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
434 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client);
435
436bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
437 ram_addr_t start,
438 ram_addr_t length);
439
440static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
441 ram_addr_t length)
442{
443 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
444 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
445 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
446}
447
448
449
450static inline
451uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
452 ram_addr_t start,
453 ram_addr_t length)
454{
455 ram_addr_t addr;
456 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
457 uint64_t num_dirty = 0;
458 unsigned long *dest = rb->bmap;
459
460
461 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
462 (start + rb->offset) &&
463 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
464 int k;
465 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
466 unsigned long * const *src;
467 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
468 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
469 DIRTY_MEMORY_BLOCK_SIZE);
470 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
471
472 src = qatomic_rcu_read(
473 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
474
475 for (k = page; k < page + nr; k++) {
476 if (src[idx][offset]) {
477 unsigned long bits = qatomic_xchg(&src[idx][offset], 0);
478 unsigned long new_dirty;
479 new_dirty = ~dest[k];
480 dest[k] |= bits;
481 new_dirty &= bits;
482 num_dirty += ctpopl(new_dirty);
483 }
484
485 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
486 offset = 0;
487 idx++;
488 }
489 }
490
491 if (rb->clear_bmap) {
492
493
494
495
496
497 clear_bmap_set(rb, start >> TARGET_PAGE_BITS,
498 length >> TARGET_PAGE_BITS);
499 } else {
500
501 memory_region_clear_dirty_bitmap(rb->mr, start, length);
502 }
503 } else {
504 ram_addr_t offset = rb->offset;
505
506 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
507 if (cpu_physical_memory_test_and_clear_dirty(
508 start + addr + offset,
509 TARGET_PAGE_SIZE,
510 DIRTY_MEMORY_MIGRATION)) {
511 long k = (start + addr) >> TARGET_PAGE_BITS;
512 if (!test_and_set_bit(k, dest)) {
513 num_dirty++;
514 }
515 }
516 }
517 }
518
519 return num_dirty;
520}
521#endif
522#endif
523