1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#ifndef RAM_ADDR_H
20#define RAM_ADDR_H
21
22#ifndef CONFIG_USER_ONLY
23#include "hw/xen/xen.h"
24
25struct RAMBlock {
26 struct rcu_head rcu;
27 struct MemoryRegion *mr;
28 uint8_t *host;
29 ram_addr_t offset;
30 ram_addr_t used_length;
31 ram_addr_t max_length;
32 void (*resized)(const char*, uint64_t length, void *host);
33 uint32_t flags;
34
35 char idstr[256];
36
37 QLIST_ENTRY(RAMBlock) next;
38 int fd;
39 size_t page_size;
40};
41
42static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
43{
44 return (b && b->host && offset < b->used_length) ? true : false;
45}
46
47static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
48{
49 assert(offset_in_ramblock(block, offset));
50 return (char *)block->host + offset;
51}
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78#define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8)
79typedef struct {
80 struct rcu_head rcu;
81 unsigned long *blocks[];
82} DirtyMemoryBlocks;
83
84typedef struct RAMList {
85 QemuMutex mutex;
86 RAMBlock *mru_block;
87
88 QLIST_HEAD(, RAMBlock) blocks;
89 DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM];
90 uint32_t version;
91} RAMList;
92extern RAMList ram_list;
93
94ram_addr_t last_ram_offset(void);
95void qemu_mutex_lock_ramlist(void);
96void qemu_mutex_unlock_ramlist(void);
97
98RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
99 bool share, const char *mem_path,
100 Error **errp);
101RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
102 MemoryRegion *mr, Error **errp);
103RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
104RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
105 void (*resized)(const char*,
106 uint64_t length,
107 void *host),
108 MemoryRegion *mr, Error **errp);
109void qemu_ram_free(RAMBlock *block);
110
111int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
112
113#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
114#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
115
116static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
117 ram_addr_t length,
118 unsigned client)
119{
120 DirtyMemoryBlocks *blocks;
121 unsigned long end, page;
122 unsigned long idx, offset, base;
123 bool dirty = false;
124
125 assert(client < DIRTY_MEMORY_NUM);
126
127 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
128 page = start >> TARGET_PAGE_BITS;
129
130 rcu_read_lock();
131
132 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
133
134 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
135 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
136 base = page - offset;
137 while (page < end) {
138 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
139 unsigned long num = next - base;
140 unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
141 if (found < num) {
142 dirty = true;
143 break;
144 }
145
146 page = next;
147 idx++;
148 offset = 0;
149 base += DIRTY_MEMORY_BLOCK_SIZE;
150 }
151
152 rcu_read_unlock();
153
154 return dirty;
155}
156
157static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
158 ram_addr_t length,
159 unsigned client)
160{
161 DirtyMemoryBlocks *blocks;
162 unsigned long end, page;
163 unsigned long idx, offset, base;
164 bool dirty = true;
165
166 assert(client < DIRTY_MEMORY_NUM);
167
168 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
169 page = start >> TARGET_PAGE_BITS;
170
171 rcu_read_lock();
172
173 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
174
175 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
176 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
177 base = page - offset;
178 while (page < end) {
179 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
180 unsigned long num = next - base;
181 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
182 if (found < num) {
183 dirty = false;
184 break;
185 }
186
187 page = next;
188 idx++;
189 offset = 0;
190 base += DIRTY_MEMORY_BLOCK_SIZE;
191 }
192
193 rcu_read_unlock();
194
195 return dirty;
196}
197
198static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
199 unsigned client)
200{
201 return cpu_physical_memory_get_dirty(addr, 1, client);
202}
203
204static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
205{
206 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
207 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
208 bool migration =
209 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
210 return !(vga && code && migration);
211}
212
213static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
214 ram_addr_t length,
215 uint8_t mask)
216{
217 uint8_t ret = 0;
218
219 if (mask & (1 << DIRTY_MEMORY_VGA) &&
220 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
221 ret |= (1 << DIRTY_MEMORY_VGA);
222 }
223 if (mask & (1 << DIRTY_MEMORY_CODE) &&
224 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
225 ret |= (1 << DIRTY_MEMORY_CODE);
226 }
227 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
228 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
229 ret |= (1 << DIRTY_MEMORY_MIGRATION);
230 }
231 return ret;
232}
233
234static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
235 unsigned client)
236{
237 unsigned long page, idx, offset;
238 DirtyMemoryBlocks *blocks;
239
240 assert(client < DIRTY_MEMORY_NUM);
241
242 page = addr >> TARGET_PAGE_BITS;
243 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
244 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
245
246 rcu_read_lock();
247
248 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
249
250 set_bit_atomic(offset, blocks->blocks[idx]);
251
252 rcu_read_unlock();
253}
254
255static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
256 ram_addr_t length,
257 uint8_t mask)
258{
259 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
260 unsigned long end, page;
261 unsigned long idx, offset, base;
262 int i;
263
264 if (!mask && !xen_enabled()) {
265 return;
266 }
267
268 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
269 page = start >> TARGET_PAGE_BITS;
270
271 rcu_read_lock();
272
273 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
274 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
275 }
276
277 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
278 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
279 base = page - offset;
280 while (page < end) {
281 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
282
283 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
284 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
285 offset, next - page);
286 }
287 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
288 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
289 offset, next - page);
290 }
291 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
292 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
293 offset, next - page);
294 }
295
296 page = next;
297 idx++;
298 offset = 0;
299 base += DIRTY_MEMORY_BLOCK_SIZE;
300 }
301
302 rcu_read_unlock();
303
304 xen_modified_memory(start, length);
305}
306
307#if !defined(_WIN32)
308static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
309 ram_addr_t start,
310 ram_addr_t pages)
311{
312 unsigned long i, j;
313 unsigned long page_number, c;
314 hwaddr addr;
315 ram_addr_t ram_addr;
316 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
317 unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
318 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
319
320
321 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
322 (hpratio == 1)) {
323 unsigned long **blocks[DIRTY_MEMORY_NUM];
324 unsigned long idx;
325 unsigned long offset;
326 long k;
327 long nr = BITS_TO_LONGS(pages);
328
329 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
330 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
331 DIRTY_MEMORY_BLOCK_SIZE);
332
333 rcu_read_lock();
334
335 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
336 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
337 }
338
339 for (k = 0; k < nr; k++) {
340 if (bitmap[k]) {
341 unsigned long temp = leul_to_cpu(bitmap[k]);
342
343 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
344 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
345 if (tcg_enabled()) {
346 atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
347 }
348 }
349
350 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
351 offset = 0;
352 idx++;
353 }
354 }
355
356 rcu_read_unlock();
357
358 xen_modified_memory(start, pages << TARGET_PAGE_BITS);
359 } else {
360 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
361
362
363
364
365 for (i = 0; i < len; i++) {
366 if (bitmap[i] != 0) {
367 c = leul_to_cpu(bitmap[i]);
368 do {
369 j = ctzl(c);
370 c &= ~(1ul << j);
371 page_number = (i * HOST_LONG_BITS + j) * hpratio;
372 addr = page_number * TARGET_PAGE_SIZE;
373 ram_addr = start + addr;
374 cpu_physical_memory_set_dirty_range(ram_addr,
375 TARGET_PAGE_SIZE * hpratio, clients);
376 } while (c != 0);
377 }
378 }
379 }
380}
381#endif
382
383bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
384 ram_addr_t length,
385 unsigned client);
386
387static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
388 ram_addr_t length)
389{
390 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
391 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
392 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
393}
394
395
396static inline
397uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
398 ram_addr_t start,
399 ram_addr_t length)
400{
401 ram_addr_t addr;
402 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
403 uint64_t num_dirty = 0;
404
405
406 if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
407 int k;
408 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
409 unsigned long * const *src;
410 unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
411 unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
412 DIRTY_MEMORY_BLOCK_SIZE);
413
414 rcu_read_lock();
415
416 src = atomic_rcu_read(
417 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
418
419 for (k = page; k < page + nr; k++) {
420 if (src[idx][offset]) {
421 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
422 unsigned long new_dirty;
423 new_dirty = ~dest[k];
424 dest[k] |= bits;
425 new_dirty &= bits;
426 num_dirty += ctpopl(new_dirty);
427 }
428
429 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
430 offset = 0;
431 idx++;
432 }
433 }
434
435 rcu_read_unlock();
436 } else {
437 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
438 if (cpu_physical_memory_test_and_clear_dirty(
439 start + addr,
440 TARGET_PAGE_SIZE,
441 DIRTY_MEMORY_MIGRATION)) {
442 long k = (start + addr) >> TARGET_PAGE_BITS;
443 if (!test_and_set_bit(k, dest)) {
444 num_dirty++;
445 }
446 }
447 }
448 }
449
450 return num_dirty;
451}
452
453void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
454#endif
455#endif
456
