1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#ifndef RAM_ADDR_H
20#define RAM_ADDR_H
21
22#ifndef CONFIG_USER_ONLY
23#include "cpu.h"
24#include "sysemu/xen.h"
25#include "sysemu/tcg.h"
26#include "exec/ramlist.h"
27#include "exec/ramblock.h"
28
29
30
31
32
33
34
35
36
37static inline long clear_bmap_size(uint64_t pages, uint8_t shift)
38{
39 return DIV_ROUND_UP(pages, 1UL << shift);
40}
41
42
43
44
45
46
47
48
49
50
51static inline void clear_bmap_set(RAMBlock *rb, uint64_t start,
52 uint64_t npages)
53{
54 uint8_t shift = rb->clear_bmap_shift;
55
56 bitmap_set_atomic(rb->clear_bmap, start >> shift,
57 clear_bmap_size(npages, shift));
58}
59
60
61
62
63
64
65
66
67
68static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page)
69{
70 uint8_t shift = rb->clear_bmap_shift;
71
72 return bitmap_test_and_clear_atomic(rb->clear_bmap, page >> shift, 1);
73}
74
75static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
76{
77 return (b && b->host && offset < b->used_length) ? true : false;
78}
79
80static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
81{
82 assert(offset_in_ramblock(block, offset));
83 return (char *)block->host + offset;
84}
85
86static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
87 RAMBlock *rb)
88{
89 uint64_t host_addr_offset =
90 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
91 return host_addr_offset >> TARGET_PAGE_BITS;
92}
93
94bool ramblock_is_pmem(RAMBlock *rb);
95
96long qemu_minrampagesize(void);
97long qemu_maxrampagesize(void);
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
121 uint32_t ram_flags, const char *mem_path,
122 bool readonly, Error **errp);
123RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
124 uint32_t ram_flags, int fd, off_t offset,
125 bool readonly, Error **errp);
126
127RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
128 MemoryRegion *mr, Error **errp);
129RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share, MemoryRegion *mr,
130 Error **errp);
131RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
132 void (*resized)(const char*,
133 uint64_t length,
134 void *host),
135 MemoryRegion *mr, Error **errp);
136void qemu_ram_free(RAMBlock *block);
137
138int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
139
140void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length);
141
142
143static inline void qemu_ram_block_writeback(RAMBlock *block)
144{
145 qemu_ram_msync(block, 0, block->used_length);
146}
147
148#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
149#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
150
151void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end);
152
153static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
154 ram_addr_t length,
155 unsigned client)
156{
157 DirtyMemoryBlocks *blocks;
158 unsigned long end, page;
159 unsigned long idx, offset, base;
160 bool dirty = false;
161
162 assert(client < DIRTY_MEMORY_NUM);
163
164 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
165 page = start >> TARGET_PAGE_BITS;
166
167 WITH_RCU_READ_LOCK_GUARD() {
168 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
169
170 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
171 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
172 base = page - offset;
173 while (page < end) {
174 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
175 unsigned long num = next - base;
176 unsigned long found = find_next_bit(blocks->blocks[idx],
177 num, offset);
178 if (found < num) {
179 dirty = true;
180 break;
181 }
182
183 page = next;
184 idx++;
185 offset = 0;
186 base += DIRTY_MEMORY_BLOCK_SIZE;
187 }
188 }
189
190 return dirty;
191}
192
193static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
194 ram_addr_t length,
195 unsigned client)
196{
197 DirtyMemoryBlocks *blocks;
198 unsigned long end, page;
199 unsigned long idx, offset, base;
200 bool dirty = true;
201
202 assert(client < DIRTY_MEMORY_NUM);
203
204 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
205 page = start >> TARGET_PAGE_BITS;
206
207 RCU_READ_LOCK_GUARD();
208
209 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
210
211 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
212 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
213 base = page - offset;
214 while (page < end) {
215 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
216 unsigned long num = next - base;
217 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
218 if (found < num) {
219 dirty = false;
220 break;
221 }
222
223 page = next;
224 idx++;
225 offset = 0;
226 base += DIRTY_MEMORY_BLOCK_SIZE;
227 }
228
229 return dirty;
230}
231
232static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
233 unsigned client)
234{
235 return cpu_physical_memory_get_dirty(addr, 1, client);
236}
237
238static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
239{
240 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
241 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
242 bool migration =
243 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
244 return !(vga && code && migration);
245}
246
247static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
248 ram_addr_t length,
249 uint8_t mask)
250{
251 uint8_t ret = 0;
252
253 if (mask & (1 << DIRTY_MEMORY_VGA) &&
254 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
255 ret |= (1 << DIRTY_MEMORY_VGA);
256 }
257 if (mask & (1 << DIRTY_MEMORY_CODE) &&
258 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
259 ret |= (1 << DIRTY_MEMORY_CODE);
260 }
261 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
262 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
263 ret |= (1 << DIRTY_MEMORY_MIGRATION);
264 }
265 return ret;
266}
267
268static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
269 unsigned client)
270{
271 unsigned long page, idx, offset;
272 DirtyMemoryBlocks *blocks;
273
274 assert(client < DIRTY_MEMORY_NUM);
275
276 page = addr >> TARGET_PAGE_BITS;
277 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
278 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
279
280 RCU_READ_LOCK_GUARD();
281
282 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
283
284 set_bit_atomic(offset, blocks->blocks[idx]);
285}
286
287static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
288 ram_addr_t length,
289 uint8_t mask)
290{
291 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
292 unsigned long end, page;
293 unsigned long idx, offset, base;
294 int i;
295
296 if (!mask && !xen_enabled()) {
297 return;
298 }
299
300 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
301 page = start >> TARGET_PAGE_BITS;
302
303 WITH_RCU_READ_LOCK_GUARD() {
304 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
305 blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]);
306 }
307
308 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
309 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
310 base = page - offset;
311 while (page < end) {
312 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
313
314 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
315 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
316 offset, next - page);
317 }
318 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
319 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
320 offset, next - page);
321 }
322 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
323 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
324 offset, next - page);
325 }
326
327 page = next;
328 idx++;
329 offset = 0;
330 base += DIRTY_MEMORY_BLOCK_SIZE;
331 }
332 }
333
334 xen_hvm_modified_memory(start, length);
335}
336
337#if !defined(_WIN32)
338static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
339 ram_addr_t start,
340 ram_addr_t pages)
341{
342 unsigned long i, j;
343 unsigned long page_number, c;
344 hwaddr addr;
345 ram_addr_t ram_addr;
346 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
347 unsigned long hpratio = qemu_real_host_page_size / TARGET_PAGE_SIZE;
348 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
349
350
351 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
352 (hpratio == 1)) {
353 unsigned long **blocks[DIRTY_MEMORY_NUM];
354 unsigned long idx;
355 unsigned long offset;
356 long k;
357 long nr = BITS_TO_LONGS(pages);
358
359 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
360 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
361 DIRTY_MEMORY_BLOCK_SIZE);
362
363 WITH_RCU_READ_LOCK_GUARD() {
364 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
365 blocks[i] =
366 qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
367 }
368
369 for (k = 0; k < nr; k++) {
370 if (bitmap[k]) {
371 unsigned long temp = leul_to_cpu(bitmap[k]);
372
373 qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
374
375 if (global_dirty_log) {
376 qatomic_or(
377 &blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
378 temp);
379 }
380
381 if (tcg_enabled()) {
382 qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
383 temp);
384 }
385 }
386
387 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
388 offset = 0;
389 idx++;
390 }
391 }
392 }
393
394 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
395 } else {
396 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
397
398 if (!global_dirty_log) {
399 clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
400 }
401
402
403
404
405
406 for (i = 0; i < len; i++) {
407 if (bitmap[i] != 0) {
408 c = leul_to_cpu(bitmap[i]);
409 do {
410 j = ctzl(c);
411 c &= ~(1ul << j);
412 page_number = (i * HOST_LONG_BITS + j) * hpratio;
413 addr = page_number * TARGET_PAGE_SIZE;
414 ram_addr = start + addr;
415 cpu_physical_memory_set_dirty_range(ram_addr,
416 TARGET_PAGE_SIZE * hpratio, clients);
417 } while (c != 0);
418 }
419 }
420 }
421}
422#endif
423
424bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
425 ram_addr_t length,
426 unsigned client);
427
428DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
429 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client);
430
431bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
432 ram_addr_t start,
433 ram_addr_t length);
434
435static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
436 ram_addr_t length)
437{
438 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
439 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
440 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
441}
442
443
444
445static inline
446uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
447 ram_addr_t start,
448 ram_addr_t length)
449{
450 ram_addr_t addr;
451 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
452 uint64_t num_dirty = 0;
453 unsigned long *dest = rb->bmap;
454
455
456 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
457 (start + rb->offset) &&
458 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
459 int k;
460 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
461 unsigned long * const *src;
462 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
463 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
464 DIRTY_MEMORY_BLOCK_SIZE);
465 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
466
467 src = qatomic_rcu_read(
468 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
469
470 for (k = page; k < page + nr; k++) {
471 if (src[idx][offset]) {
472 unsigned long bits = qatomic_xchg(&src[idx][offset], 0);
473 unsigned long new_dirty;
474 new_dirty = ~dest[k];
475 dest[k] |= bits;
476 new_dirty &= bits;
477 num_dirty += ctpopl(new_dirty);
478 }
479
480 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
481 offset = 0;
482 idx++;
483 }
484 }
485
486 if (rb->clear_bmap) {
487
488
489
490
491
492 clear_bmap_set(rb, start >> TARGET_PAGE_BITS,
493 length >> TARGET_PAGE_BITS);
494 } else {
495
496 memory_region_clear_dirty_bitmap(rb->mr, start, length);
497 }
498 } else {
499 ram_addr_t offset = rb->offset;
500
501 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
502 if (cpu_physical_memory_test_and_clear_dirty(
503 start + addr + offset,
504 TARGET_PAGE_SIZE,
505 DIRTY_MEMORY_MIGRATION)) {
506 long k = (start + addr) >> TARGET_PAGE_BITS;
507 if (!test_and_set_bit(k, dest)) {
508 num_dirty++;
509 }
510 }
511 }
512 }
513
514 return num_dirty;
515}
516#endif
517#endif
518