1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include "qemu/osdep.h"
20#include "qapi/error.h"
21
22#include "qemu/cutils.h"
23#include "cpu.h"
24#include "exec/exec-all.h"
25#include "exec/target_page.h"
26#include "tcg.h"
27#include "hw/qdev-core.h"
28#include "hw/qdev-properties.h"
29#if !defined(CONFIG_USER_ONLY)
30#include "hw/boards.h"
31#include "hw/xen/xen.h"
32#endif
33#include "sysemu/kvm.h"
34#include "sysemu/sysemu.h"
35#include "qemu/timer.h"
36#include "qemu/config-file.h"
37#include "qemu/error-report.h"
38#if defined(CONFIG_USER_ONLY)
39#include "qemu.h"
40#else
41#include "hw/hw.h"
42#include "exec/memory.h"
43#include "exec/ioport.h"
44#include "sysemu/dma.h"
45#include "sysemu/numa.h"
46#include "sysemu/hw_accel.h"
47#include "exec/address-spaces.h"
48#include "sysemu/xen-mapcache.h"
49#include "trace-root.h"
50
51#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
52#include <linux/falloc.h>
53#endif
54
55#endif
56#include "qemu/rcu_queue.h"
57#include "qemu/main-loop.h"
58#include "translate-all.h"
59#include "sysemu/replay.h"
60
61#include "exec/memory-internal.h"
62#include "exec/ram_addr.h"
63#include "exec/log.h"
64
65#include "migration/vmstate.h"
66
67#include "qemu/range.h"
68#ifndef _WIN32
69#include "qemu/mmap-alloc.h"
70#endif
71
72#include "monitor/monitor.h"
73
74
75
76#if !defined(CONFIG_USER_ONLY)
77
78
79
80RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
81
82static MemoryRegion *system_memory;
83static MemoryRegion *system_io;
84
85AddressSpace address_space_io;
86AddressSpace address_space_memory;
87
88MemoryRegion io_mem_rom, io_mem_notdirty;
89static MemoryRegion io_mem_unassigned;
90#endif
91
92#ifdef TARGET_PAGE_BITS_VARY
93int target_page_bits;
94bool target_page_bits_decided;
95#endif
96
97struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
98
99
100__thread CPUState *current_cpu;
101
102
103
104int use_icount;
105
106uintptr_t qemu_host_page_size;
107intptr_t qemu_host_page_mask;
108
109bool set_preferred_target_page_bits(int bits)
110{
111
112
113
114
115
116#ifdef TARGET_PAGE_BITS_VARY
117 assert(bits >= TARGET_PAGE_BITS_MIN);
118 if (target_page_bits == 0 || target_page_bits > bits) {
119 if (target_page_bits_decided) {
120 return false;
121 }
122 target_page_bits = bits;
123 }
124#endif
125 return true;
126}
127
128#if !defined(CONFIG_USER_ONLY)
129
130static void finalize_target_page_bits(void)
131{
132#ifdef TARGET_PAGE_BITS_VARY
133 if (target_page_bits == 0) {
134 target_page_bits = TARGET_PAGE_BITS_MIN;
135 }
136 target_page_bits_decided = true;
137#endif
138}
139
140typedef struct PhysPageEntry PhysPageEntry;
141
142struct PhysPageEntry {
143
144 uint32_t skip : 6;
145
146 uint32_t ptr : 26;
147};
148
149#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
150
151
152#define ADDR_SPACE_BITS 64
153
154#define P_L2_BITS 9
155#define P_L2_SIZE (1 << P_L2_BITS)
156
157#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
158
159typedef PhysPageEntry Node[P_L2_SIZE];
160
161typedef struct PhysPageMap {
162 struct rcu_head rcu;
163
164 unsigned sections_nb;
165 unsigned sections_nb_alloc;
166 unsigned nodes_nb;
167 unsigned nodes_nb_alloc;
168 Node *nodes;
169 MemoryRegionSection *sections;
170} PhysPageMap;
171
172struct AddressSpaceDispatch {
173 MemoryRegionSection *mru_section;
174
175
176
177 PhysPageEntry phys_map;
178 PhysPageMap map;
179};
180
181#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
182typedef struct subpage_t {
183 MemoryRegion iomem;
184 FlatView *fv;
185 hwaddr base;
186 uint16_t sub_section[];
187} subpage_t;
188
189#define PHYS_SECTION_UNASSIGNED 0
190#define PHYS_SECTION_NOTDIRTY 1
191#define PHYS_SECTION_ROM 2
192#define PHYS_SECTION_WATCH 3
193
194static void io_mem_init(void);
195static void memory_map_init(void);
196static void tcg_commit(MemoryListener *listener);
197
198static MemoryRegion io_mem_watch;
199
200
201
202
203
204
205
206
207struct CPUAddressSpace {
208 CPUState *cpu;
209 AddressSpace *as;
210 struct AddressSpaceDispatch *memory_dispatch;
211 MemoryListener tcg_as_listener;
212};
213
214struct DirtyBitmapSnapshot {
215 ram_addr_t start;
216 ram_addr_t end;
217 unsigned long dirty[];
218};
219
220#endif
221
222#if !defined(CONFIG_USER_ONLY)
223
224static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
225{
226 static unsigned alloc_hint = 16;
227 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
228 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
229 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
230 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
231 alloc_hint = map->nodes_nb_alloc;
232 }
233}
234
235static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
236{
237 unsigned i;
238 uint32_t ret;
239 PhysPageEntry e;
240 PhysPageEntry *p;
241
242 ret = map->nodes_nb++;
243 p = map->nodes[ret];
244 assert(ret != PHYS_MAP_NODE_NIL);
245 assert(ret != map->nodes_nb_alloc);
246
247 e.skip = leaf ? 0 : 1;
248 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
249 for (i = 0; i < P_L2_SIZE; ++i) {
250 memcpy(&p[i], &e, sizeof(e));
251 }
252 return ret;
253}
254
255static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
256 hwaddr *index, hwaddr *nb, uint16_t leaf,
257 int level)
258{
259 PhysPageEntry *p;
260 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
261
262 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
263 lp->ptr = phys_map_node_alloc(map, level == 0);
264 }
265 p = map->nodes[lp->ptr];
266 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
267
268 while (*nb && lp < &p[P_L2_SIZE]) {
269 if ((*index & (step - 1)) == 0 && *nb >= step) {
270 lp->skip = 0;
271 lp->ptr = leaf;
272 *index += step;
273 *nb -= step;
274 } else {
275 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
276 }
277 ++lp;
278 }
279}
280
281static void phys_page_set(AddressSpaceDispatch *d,
282 hwaddr index, hwaddr nb,
283 uint16_t leaf)
284{
285
286 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
287
288 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
289}
290
291
292
293
294static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
295{
296 unsigned valid_ptr = P_L2_SIZE;
297 int valid = 0;
298 PhysPageEntry *p;
299 int i;
300
301 if (lp->ptr == PHYS_MAP_NODE_NIL) {
302 return;
303 }
304
305 p = nodes[lp->ptr];
306 for (i = 0; i < P_L2_SIZE; i++) {
307 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
308 continue;
309 }
310
311 valid_ptr = i;
312 valid++;
313 if (p[i].skip) {
314 phys_page_compact(&p[i], nodes);
315 }
316 }
317
318
319 if (valid != 1) {
320 return;
321 }
322
323 assert(valid_ptr < P_L2_SIZE);
324
325
326 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
327 return;
328 }
329
330 lp->ptr = p[valid_ptr].ptr;
331 if (!p[valid_ptr].skip) {
332
333
334
335
336
337
338 lp->skip = 0;
339 } else {
340 lp->skip += p[valid_ptr].skip;
341 }
342}
343
344void address_space_dispatch_compact(AddressSpaceDispatch *d)
345{
346 if (d->phys_map.skip) {
347 phys_page_compact(&d->phys_map, d->map.nodes);
348 }
349}
350
351static inline bool section_covers_addr(const MemoryRegionSection *section,
352 hwaddr addr)
353{
354
355
356
357 return int128_gethi(section->size) ||
358 range_covers_byte(section->offset_within_address_space,
359 int128_getlo(section->size), addr);
360}
361
362static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
363{
364 PhysPageEntry lp = d->phys_map, *p;
365 Node *nodes = d->map.nodes;
366 MemoryRegionSection *sections = d->map.sections;
367 hwaddr index = addr >> TARGET_PAGE_BITS;
368 int i;
369
370 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
371 if (lp.ptr == PHYS_MAP_NODE_NIL) {
372 return §ions[PHYS_SECTION_UNASSIGNED];
373 }
374 p = nodes[lp.ptr];
375 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
376 }
377
378 if (section_covers_addr(§ions[lp.ptr], addr)) {
379 return §ions[lp.ptr];
380 } else {
381 return §ions[PHYS_SECTION_UNASSIGNED];
382 }
383}
384
385
386static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
387 hwaddr addr,
388 bool resolve_subpage)
389{
390 MemoryRegionSection *section = atomic_read(&d->mru_section);
391 subpage_t *subpage;
392
393 if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
394 !section_covers_addr(section, addr)) {
395 section = phys_page_find(d, addr);
396 atomic_set(&d->mru_section, section);
397 }
398 if (resolve_subpage && section->mr->subpage) {
399 subpage = container_of(section->mr, subpage_t, iomem);
400 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
401 }
402 return section;
403}
404
405
406static MemoryRegionSection *
407address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
408 hwaddr *plen, bool resolve_subpage)
409{
410 MemoryRegionSection *section;
411 MemoryRegion *mr;
412 Int128 diff;
413
414 section = address_space_lookup_region(d, addr, resolve_subpage);
415
416 addr -= section->offset_within_address_space;
417
418
419 *xlat = addr + section->offset_within_region;
420
421 mr = section->mr;
422
423
424
425
426
427
428
429
430
431
432
433
434 if (memory_region_is_ram(mr)) {
435 diff = int128_sub(section->size, int128_make64(addr));
436 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
437 }
438 return section;
439}
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
464 hwaddr *xlat,
465 hwaddr *plen_out,
466 hwaddr *page_mask_out,
467 bool is_write,
468 bool is_mmio,
469 AddressSpace **target_as,
470 MemTxAttrs attrs)
471{
472 MemoryRegionSection *section;
473 hwaddr page_mask = (hwaddr)-1;
474
475 do {
476 hwaddr addr = *xlat;
477 IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
478 int iommu_idx = 0;
479 IOMMUTLBEntry iotlb;
480
481 if (imrc->attrs_to_index) {
482 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
483 }
484
485 iotlb = imrc->translate(iommu_mr, addr, is_write ?
486 IOMMU_WO : IOMMU_RO, iommu_idx);
487
488 if (!(iotlb.perm & (1 << is_write))) {
489 goto unassigned;
490 }
491
492 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
493 | (addr & iotlb.addr_mask));
494 page_mask &= iotlb.addr_mask;
495 *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
496 *target_as = iotlb.target_as;
497
498 section = address_space_translate_internal(
499 address_space_to_dispatch(iotlb.target_as), addr, xlat,
500 plen_out, is_mmio);
501
502 iommu_mr = memory_region_get_iommu(section->mr);
503 } while (unlikely(iommu_mr));
504
505 if (page_mask_out) {
506 *page_mask_out = page_mask;
507 }
508 return *section;
509
510unassigned:
511 return (MemoryRegionSection) { .mr = &io_mem_unassigned };
512}
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534static MemoryRegionSection flatview_do_translate(FlatView *fv,
535 hwaddr addr,
536 hwaddr *xlat,
537 hwaddr *plen_out,
538 hwaddr *page_mask_out,
539 bool is_write,
540 bool is_mmio,
541 AddressSpace **target_as,
542 MemTxAttrs attrs)
543{
544 MemoryRegionSection *section;
545 IOMMUMemoryRegion *iommu_mr;
546 hwaddr plen = (hwaddr)(-1);
547
548 if (!plen_out) {
549 plen_out = &plen;
550 }
551
552 section = address_space_translate_internal(
553 flatview_to_dispatch(fv), addr, xlat,
554 plen_out, is_mmio);
555
556 iommu_mr = memory_region_get_iommu(section->mr);
557 if (unlikely(iommu_mr)) {
558 return address_space_translate_iommu(iommu_mr, xlat,
559 plen_out, page_mask_out,
560 is_write, is_mmio,
561 target_as, attrs);
562 }
563 if (page_mask_out) {
564
565 *page_mask_out = ~TARGET_PAGE_MASK;
566 }
567
568 return *section;
569}
570
571
572IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
573 bool is_write, MemTxAttrs attrs)
574{
575 MemoryRegionSection section;
576 hwaddr xlat, page_mask;
577
578
579
580
581
582 section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
583 NULL, &page_mask, is_write, false, &as,
584 attrs);
585
586
587 if (section.mr == &io_mem_unassigned) {
588 goto iotlb_fail;
589 }
590
591
592 xlat += section.offset_within_address_space -
593 section.offset_within_region;
594
595 return (IOMMUTLBEntry) {
596 .target_as = as,
597 .iova = addr & ~page_mask,
598 .translated_addr = xlat & ~page_mask,
599 .addr_mask = page_mask,
600
601 .perm = IOMMU_RW,
602 };
603
604iotlb_fail:
605 return (IOMMUTLBEntry) {0};
606}
607
608
609MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
610 hwaddr *plen, bool is_write,
611 MemTxAttrs attrs)
612{
613 MemoryRegion *mr;
614 MemoryRegionSection section;
615 AddressSpace *as = NULL;
616
617
618 section = flatview_do_translate(fv, addr, xlat, plen, NULL,
619 is_write, true, &as, attrs);
620 mr = section.mr;
621
622 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
623 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
624 *plen = MIN(page, *plen);
625 }
626
627 return mr;
628}
629
630typedef struct TCGIOMMUNotifier {
631 IOMMUNotifier n;
632 MemoryRegion *mr;
633 CPUState *cpu;
634 int iommu_idx;
635 bool active;
636} TCGIOMMUNotifier;
637
638static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
639{
640 TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
641
642 if (!notifier->active) {
643 return;
644 }
645 tlb_flush(notifier->cpu);
646 notifier->active = false;
647
648
649
650
651
652}
653
654static void tcg_register_iommu_notifier(CPUState *cpu,
655 IOMMUMemoryRegion *iommu_mr,
656 int iommu_idx)
657{
658
659
660
661
662 MemoryRegion *mr = MEMORY_REGION(iommu_mr);
663 TCGIOMMUNotifier *notifier;
664 int i;
665
666 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
667 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
668 if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
669 break;
670 }
671 }
672 if (i == cpu->iommu_notifiers->len) {
673
674 cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
675 notifier = g_new0(TCGIOMMUNotifier, 1);
676 g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
677
678 notifier->mr = mr;
679 notifier->iommu_idx = iommu_idx;
680 notifier->cpu = cpu;
681
682
683
684
685
686
687 iommu_notifier_init(¬ifier->n,
688 tcg_iommu_unmap_notify,
689 IOMMU_NOTIFIER_UNMAP,
690 0,
691 HWADDR_MAX,
692 iommu_idx);
693 memory_region_register_iommu_notifier(notifier->mr, ¬ifier->n);
694 }
695
696 if (!notifier->active) {
697 notifier->active = true;
698 }
699}
700
701static void tcg_iommu_free_notifier_list(CPUState *cpu)
702{
703
704 int i;
705 TCGIOMMUNotifier *notifier;
706
707 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
708 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
709 memory_region_unregister_iommu_notifier(notifier->mr, ¬ifier->n);
710 g_free(notifier);
711 }
712 g_array_free(cpu->iommu_notifiers, true);
713}
714
715
716MemoryRegionSection *
717address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
718 hwaddr *xlat, hwaddr *plen,
719 MemTxAttrs attrs, int *prot)
720{
721 MemoryRegionSection *section;
722 IOMMUMemoryRegion *iommu_mr;
723 IOMMUMemoryRegionClass *imrc;
724 IOMMUTLBEntry iotlb;
725 int iommu_idx;
726 AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
727
728 for (;;) {
729 section = address_space_translate_internal(d, addr, &addr, plen, false);
730
731 iommu_mr = memory_region_get_iommu(section->mr);
732 if (!iommu_mr) {
733 break;
734 }
735
736 imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
737
738 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
739 tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
740
741
742
743 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
744 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
745 | (addr & iotlb.addr_mask));
746
747
748
749
750 if (!(iotlb.perm & IOMMU_RO)) {
751 *prot &= ~(PAGE_READ | PAGE_EXEC);
752 }
753 if (!(iotlb.perm & IOMMU_WO)) {
754 *prot &= ~PAGE_WRITE;
755 }
756
757 if (!*prot) {
758 goto translate_fail;
759 }
760
761 d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
762 }
763
764 assert(!memory_region_is_iommu(section->mr));
765 *xlat = addr;
766 return section;
767
768translate_fail:
769 return &d->map.sections[PHYS_SECTION_UNASSIGNED];
770}
771#endif
772
773#if !defined(CONFIG_USER_ONLY)
774
775static int cpu_common_post_load(void *opaque, int version_id)
776{
777 CPUState *cpu = opaque;
778
779
780
781 cpu->interrupt_request &= ~0x01;
782 tlb_flush(cpu);
783
784
785
786
787
788
789 tb_flush(cpu);
790
791 return 0;
792}
793
794static int cpu_common_pre_load(void *opaque)
795{
796 CPUState *cpu = opaque;
797
798 cpu->exception_index = -1;
799
800 return 0;
801}
802
803static bool cpu_common_exception_index_needed(void *opaque)
804{
805 CPUState *cpu = opaque;
806
807 return tcg_enabled() && cpu->exception_index != -1;
808}
809
810static const VMStateDescription vmstate_cpu_common_exception_index = {
811 .name = "cpu_common/exception_index",
812 .version_id = 1,
813 .minimum_version_id = 1,
814 .needed = cpu_common_exception_index_needed,
815 .fields = (VMStateField[]) {
816 VMSTATE_INT32(exception_index, CPUState),
817 VMSTATE_END_OF_LIST()
818 }
819};
820
821static bool cpu_common_crash_occurred_needed(void *opaque)
822{
823 CPUState *cpu = opaque;
824
825 return cpu->crash_occurred;
826}
827
828static const VMStateDescription vmstate_cpu_common_crash_occurred = {
829 .name = "cpu_common/crash_occurred",
830 .version_id = 1,
831 .minimum_version_id = 1,
832 .needed = cpu_common_crash_occurred_needed,
833 .fields = (VMStateField[]) {
834 VMSTATE_BOOL(crash_occurred, CPUState),
835 VMSTATE_END_OF_LIST()
836 }
837};
838
839const VMStateDescription vmstate_cpu_common = {
840 .name = "cpu_common",
841 .version_id = 1,
842 .minimum_version_id = 1,
843 .pre_load = cpu_common_pre_load,
844 .post_load = cpu_common_post_load,
845 .fields = (VMStateField[]) {
846 VMSTATE_UINT32(halted, CPUState),
847 VMSTATE_UINT32(interrupt_request, CPUState),
848 VMSTATE_END_OF_LIST()
849 },
850 .subsections = (const VMStateDescription*[]) {
851 &vmstate_cpu_common_exception_index,
852 &vmstate_cpu_common_crash_occurred,
853 NULL
854 }
855};
856
857#endif
858
859CPUState *qemu_get_cpu(int index)
860{
861 CPUState *cpu;
862
863 CPU_FOREACH(cpu) {
864 if (cpu->cpu_index == index) {
865 return cpu;
866 }
867 }
868
869 return NULL;
870}
871
872#if !defined(CONFIG_USER_ONLY)
873void cpu_address_space_init(CPUState *cpu, int asidx,
874 const char *prefix, MemoryRegion *mr)
875{
876 CPUAddressSpace *newas;
877 AddressSpace *as = g_new0(AddressSpace, 1);
878 char *as_name;
879
880 assert(mr);
881 as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
882 address_space_init(as, mr, as_name);
883 g_free(as_name);
884
885
886 assert(asidx < cpu->num_ases);
887
888 if (asidx == 0) {
889
890 cpu->as = as;
891 }
892
893
894 assert(asidx == 0 || !kvm_enabled());
895
896 if (!cpu->cpu_ases) {
897 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
898 }
899
900 newas = &cpu->cpu_ases[asidx];
901 newas->cpu = cpu;
902 newas->as = as;
903 if (tcg_enabled()) {
904 newas->tcg_as_listener.commit = tcg_commit;
905 memory_listener_register(&newas->tcg_as_listener, as);
906 }
907}
908
909AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
910{
911
912 return cpu->cpu_ases[asidx].as;
913}
914#endif
915
916void cpu_exec_unrealizefn(CPUState *cpu)
917{
918 CPUClass *cc = CPU_GET_CLASS(cpu);
919
920 cpu_list_remove(cpu);
921
922 if (cc->vmsd != NULL) {
923 vmstate_unregister(NULL, cc->vmsd, cpu);
924 }
925 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
926 vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
927 }
928#ifndef CONFIG_USER_ONLY
929 tcg_iommu_free_notifier_list(cpu);
930#endif
931}
932
933Property cpu_common_props[] = {
934#ifndef CONFIG_USER_ONLY
935
936
937
938
939
940
941 DEFINE_PROP_LINK("memory", CPUState, memory, TYPE_MEMORY_REGION,
942 MemoryRegion *),
943#endif
944 DEFINE_PROP_END_OF_LIST(),
945};
946
947void cpu_exec_initfn(CPUState *cpu)
948{
949 cpu->as = NULL;
950 cpu->num_ases = 0;
951
952#ifndef CONFIG_USER_ONLY
953 cpu->thread_id = qemu_get_thread_id();
954 cpu->memory = system_memory;
955 object_ref(OBJECT(cpu->memory));
956#endif
957}
958
959void cpu_exec_realizefn(CPUState *cpu, Error **errp)
960{
961 CPUClass *cc = CPU_GET_CLASS(cpu);
962 static bool tcg_target_initialized;
963
964 cpu_list_add(cpu);
965
966 if (tcg_enabled() && !tcg_target_initialized) {
967 tcg_target_initialized = true;
968 cc->tcg_initialize();
969 }
970 tlb_init(cpu);
971
972#ifndef CONFIG_USER_ONLY
973 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
974 vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
975 }
976 if (cc->vmsd != NULL) {
977 vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
978 }
979
980 cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
981#endif
982}
983
984const char *parse_cpu_model(const char *cpu_model)
985{
986 ObjectClass *oc;
987 CPUClass *cc;
988 gchar **model_pieces;
989 const char *cpu_type;
990
991 model_pieces = g_strsplit(cpu_model, ",", 2);
992
993 oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
994 if (oc == NULL) {
995 error_report("unable to find CPU model '%s'", model_pieces[0]);
996 g_strfreev(model_pieces);
997 exit(EXIT_FAILURE);
998 }
999
1000 cpu_type = object_class_get_name(oc);
1001 cc = CPU_CLASS(oc);
1002 cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
1003 g_strfreev(model_pieces);
1004 return cpu_type;
1005}
1006
1007#if defined(CONFIG_USER_ONLY)
1008void tb_invalidate_phys_addr(target_ulong addr)
1009{
1010 mmap_lock();
1011 tb_invalidate_phys_page_range(addr, addr + 1, 0);
1012 mmap_unlock();
1013}
1014
1015static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
1016{
1017 tb_invalidate_phys_addr(pc);
1018}
1019#else
1020void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
1021{
1022 ram_addr_t ram_addr;
1023 MemoryRegion *mr;
1024 hwaddr l = 1;
1025
1026 if (!tcg_enabled()) {
1027 return;
1028 }
1029
1030 rcu_read_lock();
1031 mr = address_space_translate(as, addr, &addr, &l, false, attrs);
1032 if (!(memory_region_is_ram(mr)
1033 || memory_region_is_romd(mr))) {
1034 rcu_read_unlock();
1035 return;
1036 }
1037 ram_addr = memory_region_get_ram_addr(mr) + addr;
1038 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1039 rcu_read_unlock();
1040}
1041
1042static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
1043{
1044 MemTxAttrs attrs;
1045 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
1046 int asidx = cpu_asidx_from_attrs(cpu, attrs);
1047 if (phys != -1) {
1048
1049 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
1050 phys | (pc & ~TARGET_PAGE_MASK), attrs);
1051 }
1052}
1053#endif
1054
1055#if defined(CONFIG_USER_ONLY)
1056void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
1057
1058{
1059}
1060
1061int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
1062 int flags)
1063{
1064 return -ENOSYS;
1065}
1066
1067void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
1068{
1069}
1070
1071int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
1072 int flags, CPUWatchpoint **watchpoint)
1073{
1074 return -ENOSYS;
1075}
1076#else
1077
1078int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
1079 int flags, CPUWatchpoint **watchpoint)
1080{
1081 CPUWatchpoint *wp;
1082
1083
1084 if (len == 0 || (addr + len - 1) < addr) {
1085 error_report("tried to set invalid watchpoint at %"
1086 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
1087 return -EINVAL;
1088 }
1089 wp = g_malloc(sizeof(*wp));
1090
1091 wp->vaddr = addr;
1092 wp->len = len;
1093 wp->flags = flags;
1094
1095
1096 if (flags & BP_GDB) {
1097 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
1098 } else {
1099 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
1100 }
1101
1102 tlb_flush_page(cpu, addr);
1103
1104 if (watchpoint)
1105 *watchpoint = wp;
1106 return 0;
1107}
1108
1109
1110int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
1111 int flags)
1112{
1113 CPUWatchpoint *wp;
1114
1115 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1116 if (addr == wp->vaddr && len == wp->len
1117 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1118 cpu_watchpoint_remove_by_ref(cpu, wp);
1119 return 0;
1120 }
1121 }
1122 return -ENOENT;
1123}
1124
1125
1126void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
1127{
1128 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
1129
1130 tlb_flush_page(cpu, watchpoint->vaddr);
1131
1132 g_free(watchpoint);
1133}
1134
1135
1136void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
1137{
1138 CPUWatchpoint *wp, *next;
1139
1140 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
1141 if (wp->flags & mask) {
1142 cpu_watchpoint_remove_by_ref(cpu, wp);
1143 }
1144 }
1145}
1146
1147
1148
1149
1150
1151
1152static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
1153 vaddr addr,
1154 vaddr len)
1155{
1156
1157
1158
1159
1160
1161 vaddr wpend = wp->vaddr + wp->len - 1;
1162 vaddr addrend = addr + len - 1;
1163
1164 return !(addr > wpend || wp->vaddr > addrend);
1165}
1166
1167#endif
1168
1169
1170int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
1171 CPUBreakpoint **breakpoint)
1172{
1173 CPUBreakpoint *bp;
1174
1175 bp = g_malloc(sizeof(*bp));
1176
1177 bp->pc = pc;
1178 bp->flags = flags;
1179
1180
1181 if (flags & BP_GDB) {
1182 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
1183 } else {
1184 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
1185 }
1186
1187 breakpoint_invalidate(cpu, pc);
1188
1189 if (breakpoint) {
1190 *breakpoint = bp;
1191 }
1192 return 0;
1193}
1194
1195
1196int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
1197{
1198 CPUBreakpoint *bp;
1199
1200 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
1201 if (bp->pc == pc && bp->flags == flags) {
1202 cpu_breakpoint_remove_by_ref(cpu, bp);
1203 return 0;
1204 }
1205 }
1206 return -ENOENT;
1207}
1208
1209
1210void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
1211{
1212 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
1213
1214 breakpoint_invalidate(cpu, breakpoint->pc);
1215
1216 g_free(breakpoint);
1217}
1218
1219
1220void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
1221{
1222 CPUBreakpoint *bp, *next;
1223
1224 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
1225 if (bp->flags & mask) {
1226 cpu_breakpoint_remove_by_ref(cpu, bp);
1227 }
1228 }
1229}
1230
1231
1232
1233void cpu_single_step(CPUState *cpu, int enabled)
1234{
1235 if (cpu->singlestep_enabled != enabled) {
1236 cpu->singlestep_enabled = enabled;
1237 if (kvm_enabled()) {
1238 kvm_update_guest_debug(cpu, 0);
1239 } else {
1240
1241
1242 tb_flush(cpu);
1243 }
1244 }
1245}
1246
1247void cpu_abort(CPUState *cpu, const char *fmt, ...)
1248{
1249 va_list ap;
1250 va_list ap2;
1251
1252 va_start(ap, fmt);
1253 va_copy(ap2, ap);
1254 fprintf(stderr, "qemu: fatal: ");
1255 vfprintf(stderr, fmt, ap);
1256 fprintf(stderr, "\n");
1257 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
1258 if (qemu_log_separate()) {
1259 qemu_log_lock();
1260 qemu_log("qemu: fatal: ");
1261 qemu_log_vprintf(fmt, ap2);
1262 qemu_log("\n");
1263 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
1264 qemu_log_flush();
1265 qemu_log_unlock();
1266 qemu_log_close();
1267 }
1268 va_end(ap2);
1269 va_end(ap);
1270 replay_finish();
1271#if defined(CONFIG_USER_ONLY)
1272 {
1273 struct sigaction act;
1274 sigfillset(&act.sa_mask);
1275 act.sa_handler = SIG_DFL;
1276 act.sa_flags = 0;
1277 sigaction(SIGABRT, &act, NULL);
1278 }
1279#endif
1280 abort();
1281}
1282
1283#if !defined(CONFIG_USER_ONLY)
1284
1285static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1286{
1287 RAMBlock *block;
1288
1289 block = atomic_rcu_read(&ram_list.mru_block);
1290 if (block && addr - block->offset < block->max_length) {
1291 return block;
1292 }
1293 RAMBLOCK_FOREACH(block) {
1294 if (addr - block->offset < block->max_length) {
1295 goto found;
1296 }
1297 }
1298
1299 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1300 abort();
1301
1302found:
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319 ram_list.mru_block = block;
1320 return block;
1321}
1322
1323static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
1324{
1325 CPUState *cpu;
1326 ram_addr_t start1;
1327 RAMBlock *block;
1328 ram_addr_t end;
1329
1330 assert(tcg_enabled());
1331 end = TARGET_PAGE_ALIGN(start + length);
1332 start &= TARGET_PAGE_MASK;
1333
1334 rcu_read_lock();
1335 block = qemu_get_ram_block(start);
1336 assert(block == qemu_get_ram_block(end - 1));
1337 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
1338 CPU_FOREACH(cpu) {
1339 tlb_reset_dirty(cpu, start1, length);
1340 }
1341 rcu_read_unlock();
1342}
1343
1344
1345bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
1346 ram_addr_t length,
1347 unsigned client)
1348{
1349 DirtyMemoryBlocks *blocks;
1350 unsigned long end, page;
1351 bool dirty = false;
1352
1353 if (length == 0) {
1354 return false;
1355 }
1356
1357 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
1358 page = start >> TARGET_PAGE_BITS;
1359
1360 rcu_read_lock();
1361
1362 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1363
1364 while (page < end) {
1365 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1366 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1367 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1368
1369 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
1370 offset, num);
1371 page += num;
1372 }
1373
1374 rcu_read_unlock();
1375
1376 if (dirty && tcg_enabled()) {
1377 tlb_reset_dirty_range_all(start, length);
1378 }
1379
1380 return dirty;
1381}
1382
1383DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
1384 (ram_addr_t start, ram_addr_t length, unsigned client)
1385{
1386 DirtyMemoryBlocks *blocks;
1387 unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
1388 ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
1389 ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
1390 DirtyBitmapSnapshot *snap;
1391 unsigned long page, end, dest;
1392
1393 snap = g_malloc0(sizeof(*snap) +
1394 ((last - first) >> (TARGET_PAGE_BITS + 3)));
1395 snap->start = first;
1396 snap->end = last;
1397
1398 page = first >> TARGET_PAGE_BITS;
1399 end = last >> TARGET_PAGE_BITS;
1400 dest = 0;
1401
1402 rcu_read_lock();
1403
1404 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1405
1406 while (page < end) {
1407 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1408 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1409 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1410
1411 assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
1412 assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
1413 offset >>= BITS_PER_LEVEL;
1414
1415 bitmap_copy_and_clear_atomic(snap->dirty + dest,
1416 blocks->blocks[idx] + offset,
1417 num);
1418 page += num;
1419 dest += num >> BITS_PER_LEVEL;
1420 }
1421
1422 rcu_read_unlock();
1423
1424 if (tcg_enabled()) {
1425 tlb_reset_dirty_range_all(start, length);
1426 }
1427
1428 return snap;
1429}
1430
1431bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
1432 ram_addr_t start,
1433 ram_addr_t length)
1434{
1435 unsigned long page, end;
1436
1437 assert(start >= snap->start);
1438 assert(start + length <= snap->end);
1439
1440 end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
1441 page = (start - snap->start) >> TARGET_PAGE_BITS;
1442
1443 while (page < end) {
1444 if (test_bit(page, snap->dirty)) {
1445 return true;
1446 }
1447 page++;
1448 }
1449 return false;
1450}
1451
1452
1453hwaddr memory_region_section_get_iotlb(CPUState *cpu,
1454 MemoryRegionSection *section,
1455 target_ulong vaddr,
1456 hwaddr paddr, hwaddr xlat,
1457 int prot,
1458 target_ulong *address)
1459{
1460 hwaddr iotlb;
1461 CPUWatchpoint *wp;
1462
1463 if (memory_region_is_ram(section->mr)) {
1464
1465 iotlb = memory_region_get_ram_addr(section->mr) + xlat;
1466 if (!section->readonly) {
1467 iotlb |= PHYS_SECTION_NOTDIRTY;
1468 } else {
1469 iotlb |= PHYS_SECTION_ROM;
1470 }
1471 } else {
1472 AddressSpaceDispatch *d;
1473
1474 d = flatview_to_dispatch(section->fv);
1475 iotlb = section - d->map.sections;
1476 iotlb += xlat;
1477 }
1478
1479
1480
1481 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1482 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1483
1484 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1485 iotlb = PHYS_SECTION_WATCH + paddr;
1486 *address |= TLB_MMIO;
1487 break;
1488 }
1489 }
1490 }
1491
1492 return iotlb;
1493}
1494#endif
1495
1496#if !defined(CONFIG_USER_ONLY)
1497
1498static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1499 uint16_t section);
1500static subpage_t *subpage_init(FlatView *fv, hwaddr base);
1501
1502static void *(*phys_mem_alloc)(size_t size, uint64_t *align, bool shared) =
1503 qemu_anon_ram_alloc;
1504
1505
1506
1507
1508
1509
1510void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align, bool shared))
1511{
1512 phys_mem_alloc = alloc;
1513}
1514
1515static uint16_t phys_section_add(PhysPageMap *map,
1516 MemoryRegionSection *section)
1517{
1518
1519
1520
1521
1522 assert(map->sections_nb < TARGET_PAGE_SIZE);
1523
1524 if (map->sections_nb == map->sections_nb_alloc) {
1525 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1526 map->sections = g_renew(MemoryRegionSection, map->sections,
1527 map->sections_nb_alloc);
1528 }
1529 map->sections[map->sections_nb] = *section;
1530 memory_region_ref(section->mr);
1531 return map->sections_nb++;
1532}
1533
1534static void phys_section_destroy(MemoryRegion *mr)
1535{
1536 bool have_sub_page = mr->subpage;
1537
1538 memory_region_unref(mr);
1539
1540 if (have_sub_page) {
1541 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1542 object_unref(OBJECT(&subpage->iomem));
1543 g_free(subpage);
1544 }
1545}
1546
1547static void phys_sections_free(PhysPageMap *map)
1548{
1549 while (map->sections_nb > 0) {
1550 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1551 phys_section_destroy(section->mr);
1552 }
1553 g_free(map->sections);
1554 g_free(map->nodes);
1555}
1556
1557static void register_subpage(FlatView *fv, MemoryRegionSection *section)
1558{
1559 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1560 subpage_t *subpage;
1561 hwaddr base = section->offset_within_address_space
1562 & TARGET_PAGE_MASK;
1563 MemoryRegionSection *existing = phys_page_find(d, base);
1564 MemoryRegionSection subsection = {
1565 .offset_within_address_space = base,
1566 .size = int128_make64(TARGET_PAGE_SIZE),
1567 };
1568 hwaddr start, end;
1569
1570 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1571
1572 if (!(existing->mr->subpage)) {
1573 subpage = subpage_init(fv, base);
1574 subsection.fv = fv;
1575 subsection.mr = &subpage->iomem;
1576 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1577 phys_section_add(&d->map, &subsection));
1578 } else {
1579 subpage = container_of(existing->mr, subpage_t, iomem);
1580 }
1581 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1582 end = start + int128_get64(section->size) - 1;
1583 subpage_register(subpage, start, end,
1584 phys_section_add(&d->map, section));
1585}
1586
1587
1588static void register_multipage(FlatView *fv,
1589 MemoryRegionSection *section)
1590{
1591 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1592 hwaddr start_addr = section->offset_within_address_space;
1593 uint16_t section_index = phys_section_add(&d->map, section);
1594 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1595 TARGET_PAGE_BITS));
1596
1597 assert(num_pages);
1598 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1599}
1600
1601void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
1602{
1603 MemoryRegionSection now = *section, remain = *section;
1604 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1605
1606 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1607 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1608 - now.offset_within_address_space;
1609
1610 now.size = int128_min(int128_make64(left), now.size);
1611 register_subpage(fv, &now);
1612 } else {
1613 now.size = int128_zero();
1614 }
1615 while (int128_ne(remain.size, now.size)) {
1616 remain.size = int128_sub(remain.size, now.size);
1617 remain.offset_within_address_space += int128_get64(now.size);
1618 remain.offset_within_region += int128_get64(now.size);
1619 now = remain;
1620 if (int128_lt(remain.size, page_size)) {
1621 register_subpage(fv, &now);
1622 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1623 now.size = page_size;
1624 register_subpage(fv, &now);
1625 } else {
1626 now.size = int128_and(now.size, int128_neg(page_size));
1627 register_multipage(fv, &now);
1628 }
1629 }
1630}
1631
1632void qemu_flush_coalesced_mmio_buffer(void)
1633{
1634 if (kvm_enabled())
1635 kvm_flush_coalesced_mmio_buffer();
1636}
1637
1638void qemu_mutex_lock_ramlist(void)
1639{
1640 qemu_mutex_lock(&ram_list.mutex);
1641}
1642
1643void qemu_mutex_unlock_ramlist(void)
1644{
1645 qemu_mutex_unlock(&ram_list.mutex);
1646}
1647
1648void ram_block_dump(Monitor *mon)
1649{
1650 RAMBlock *block;
1651 char *psize;
1652
1653 rcu_read_lock();
1654 monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
1655 "Block Name", "PSize", "Offset", "Used", "Total");
1656 RAMBLOCK_FOREACH(block) {
1657 psize = size_to_str(block->page_size);
1658 monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
1659 " 0x%016" PRIx64 "\n", block->idstr, psize,
1660 (uint64_t)block->offset,
1661 (uint64_t)block->used_length,
1662 (uint64_t)block->max_length);
1663 g_free(psize);
1664 }
1665 rcu_read_unlock();
1666}
1667
1668#ifdef __linux__
1669
1670
1671
1672
1673
1674
1675static int find_max_supported_pagesize(Object *obj, void *opaque)
1676{
1677 long *hpsize_min = opaque;
1678
1679 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1680 long hpsize = host_memory_backend_pagesize(MEMORY_BACKEND(obj));
1681
1682 if (hpsize < *hpsize_min) {
1683 *hpsize_min = hpsize;
1684 }
1685 }
1686
1687 return 0;
1688}
1689
1690long qemu_getrampagesize(void)
1691{
1692 long hpsize = LONG_MAX;
1693 long mainrampagesize;
1694 Object *memdev_root;
1695
1696 mainrampagesize = qemu_mempath_getpagesize(mem_path);
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708 memdev_root = object_resolve_path("/objects", NULL);
1709 if (memdev_root) {
1710 object_child_foreach(memdev_root, find_max_supported_pagesize, &hpsize);
1711 }
1712 if (hpsize == LONG_MAX) {
1713
1714 return mainrampagesize;
1715 }
1716
1717
1718
1719
1720
1721 if (hpsize > mainrampagesize &&
1722 (nb_numa_nodes == 0 || numa_info[0].node_memdev == NULL)) {
1723 static bool warned;
1724 if (!warned) {
1725 error_report("Huge page support disabled (n/a for main memory).");
1726 warned = true;
1727 }
1728 return mainrampagesize;
1729 }
1730
1731 return hpsize;
1732}
1733#else
1734long qemu_getrampagesize(void)
1735{
1736 return getpagesize();
1737}
1738#endif
1739
1740#ifdef CONFIG_POSIX
1741static int64_t get_file_size(int fd)
1742{
1743 int64_t size = lseek(fd, 0, SEEK_END);
1744 if (size < 0) {
1745 return -errno;
1746 }
1747 return size;
1748}
1749
1750static int file_ram_open(const char *path,
1751 const char *region_name,
1752 bool *created,
1753 Error **errp)
1754{
1755 char *filename;
1756 char *sanitized_name;
1757 char *c;
1758 int fd = -1;
1759
1760 *created = false;
1761 for (;;) {
1762 fd = open(path, O_RDWR);
1763 if (fd >= 0) {
1764
1765 break;
1766 }
1767 if (errno == ENOENT) {
1768
1769 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1770 if (fd >= 0) {
1771 *created = true;
1772 break;
1773 }
1774 } else if (errno == EISDIR) {
1775
1776
1777 sanitized_name = g_strdup(region_name);
1778 for (c = sanitized_name; *c != '\0'; c++) {
1779 if (*c == '/') {
1780 *c = '_';
1781 }
1782 }
1783
1784 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1785 sanitized_name);
1786 g_free(sanitized_name);
1787
1788 fd = mkstemp(filename);
1789 if (fd >= 0) {
1790 unlink(filename);
1791 g_free(filename);
1792 break;
1793 }
1794 g_free(filename);
1795 }
1796 if (errno != EEXIST && errno != EINTR) {
1797 error_setg_errno(errp, errno,
1798 "can't open backing store %s for guest RAM",
1799 path);
1800 return -1;
1801 }
1802
1803
1804
1805
1806 }
1807
1808 return fd;
1809}
1810
1811static void *file_ram_alloc(RAMBlock *block,
1812 ram_addr_t memory,
1813 int fd,
1814 bool truncate,
1815 Error **errp)
1816{
1817 void *area;
1818
1819 block->page_size = qemu_fd_getpagesize(fd);
1820 if (block->mr->align % block->page_size) {
1821 error_setg(errp, "alignment 0x%" PRIx64
1822 " must be multiples of page size 0x%zx",
1823 block->mr->align, block->page_size);
1824 return NULL;
1825 } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
1826 error_setg(errp, "alignment 0x%" PRIx64
1827 " must be a power of two", block->mr->align);
1828 return NULL;
1829 }
1830 block->mr->align = MAX(block->page_size, block->mr->align);
1831#if defined(__s390x__)
1832 if (kvm_enabled()) {
1833 block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1834 }
1835#endif
1836
1837 if (memory < block->page_size) {
1838 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1839 "or larger than page size 0x%zx",
1840 memory, block->page_size);
1841 return NULL;
1842 }
1843
1844 memory = ROUND_UP(memory, block->page_size);
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860 if (truncate && ftruncate(fd, memory)) {
1861 perror("ftruncate");
1862 }
1863
1864 area = qemu_ram_mmap(fd, memory, block->mr->align,
1865 block->flags & RAM_SHARED);
1866 if (area == MAP_FAILED) {
1867 error_setg_errno(errp, errno,
1868 "unable to map backing store for guest RAM");
1869 return NULL;
1870 }
1871
1872 if (mem_prealloc) {
1873 os_mem_prealloc(fd, area, memory, smp_cpus, errp);
1874 if (errp && *errp) {
1875 qemu_ram_munmap(area, memory);
1876 return NULL;
1877 }
1878 }
1879
1880 block->fd = fd;
1881 return area;
1882}
1883#endif
1884
1885
1886
1887
1888
1889static ram_addr_t find_ram_offset(ram_addr_t size)
1890{
1891 RAMBlock *block, *next_block;
1892 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1893
1894 assert(size != 0);
1895
1896 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1897 return 0;
1898 }
1899
1900 RAMBLOCK_FOREACH(block) {
1901 ram_addr_t candidate, next = RAM_ADDR_MAX;
1902
1903
1904
1905
1906 candidate = block->offset + block->max_length;
1907 candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
1908
1909
1910
1911
1912 RAMBLOCK_FOREACH(next_block) {
1913 if (next_block->offset >= candidate) {
1914 next = MIN(next, next_block->offset);
1915 }
1916 }
1917
1918
1919
1920
1921
1922 if (next - candidate >= size && next - candidate < mingap) {
1923 offset = candidate;
1924 mingap = next - candidate;
1925 }
1926
1927 trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
1928 }
1929
1930 if (offset == RAM_ADDR_MAX) {
1931 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1932 (uint64_t)size);
1933 abort();
1934 }
1935
1936 trace_find_ram_offset(size, offset);
1937
1938 return offset;
1939}
1940
1941static unsigned long last_ram_page(void)
1942{
1943 RAMBlock *block;
1944 ram_addr_t last = 0;
1945
1946 rcu_read_lock();
1947 RAMBLOCK_FOREACH(block) {
1948 last = MAX(last, block->offset + block->max_length);
1949 }
1950 rcu_read_unlock();
1951 return last >> TARGET_PAGE_BITS;
1952}
1953
1954static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1955{
1956 int ret;
1957
1958
1959 if (!machine_dump_guest_core(current_machine)) {
1960 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1961 if (ret) {
1962 perror("qemu_madvise");
1963 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1964 "but dump_guest_core=off specified\n");
1965 }
1966 }
1967}
1968
1969const char *qemu_ram_get_idstr(RAMBlock *rb)
1970{
1971 return rb->idstr;
1972}
1973
1974bool qemu_ram_is_shared(RAMBlock *rb)
1975{
1976 return rb->flags & RAM_SHARED;
1977}
1978
1979
1980bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
1981{
1982 return rb->flags & RAM_UF_ZEROPAGE;
1983}
1984
1985void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1986{
1987 rb->flags |= RAM_UF_ZEROPAGE;
1988}
1989
1990bool qemu_ram_is_migratable(RAMBlock *rb)
1991{
1992 return rb->flags & RAM_MIGRATABLE;
1993}
1994
1995void qemu_ram_set_migratable(RAMBlock *rb)
1996{
1997 rb->flags |= RAM_MIGRATABLE;
1998}
1999
2000void qemu_ram_unset_migratable(RAMBlock *rb)
2001{
2002 rb->flags &= ~RAM_MIGRATABLE;
2003}
2004
2005
2006void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
2007{
2008 RAMBlock *block;
2009
2010 assert(new_block);
2011 assert(!new_block->idstr[0]);
2012
2013 if (dev) {
2014 char *id = qdev_get_dev_path(dev);
2015 if (id) {
2016 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2017 g_free(id);
2018 }
2019 }
2020 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2021
2022 rcu_read_lock();
2023 RAMBLOCK_FOREACH(block) {
2024 if (block != new_block &&
2025 !strcmp(block->idstr, new_block->idstr)) {
2026 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2027 new_block->idstr);
2028 abort();
2029 }
2030 }
2031 rcu_read_unlock();
2032}
2033
2034
2035void qemu_ram_unset_idstr(RAMBlock *block)
2036{
2037
2038
2039
2040
2041 if (block) {
2042 memset(block->idstr, 0, sizeof(block->idstr));
2043 }
2044}
2045
2046size_t qemu_ram_pagesize(RAMBlock *rb)
2047{
2048 return rb->page_size;
2049}
2050
2051
2052size_t qemu_ram_pagesize_largest(void)
2053{
2054 RAMBlock *block;
2055 size_t largest = 0;
2056
2057 RAMBLOCK_FOREACH(block) {
2058 largest = MAX(largest, qemu_ram_pagesize(block));
2059 }
2060
2061 return largest;
2062}
2063
2064static int memory_try_enable_merging(void *addr, size_t len)
2065{
2066 if (!machine_mem_merge(current_machine)) {
2067
2068 return 0;
2069 }
2070
2071 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
2072}
2073
2074
2075
2076
2077
2078
2079
2080
2081int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
2082{
2083 assert(block);
2084
2085 newsize = HOST_PAGE_ALIGN(newsize);
2086
2087 if (block->used_length == newsize) {
2088 return 0;
2089 }
2090
2091 if (!(block->flags & RAM_RESIZEABLE)) {
2092 error_setg_errno(errp, EINVAL,
2093 "Length mismatch: %s: 0x" RAM_ADDR_FMT
2094 " in != 0x" RAM_ADDR_FMT, block->idstr,
2095 newsize, block->used_length);
2096 return -EINVAL;
2097 }
2098
2099 if (block->max_length < newsize) {
2100 error_setg_errno(errp, EINVAL,
2101 "Length too large: %s: 0x" RAM_ADDR_FMT
2102 " > 0x" RAM_ADDR_FMT, block->idstr,
2103 newsize, block->max_length);
2104 return -EINVAL;
2105 }
2106
2107 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
2108 block->used_length = newsize;
2109 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
2110 DIRTY_CLIENTS_ALL);
2111 memory_region_set_size(block->mr, newsize);
2112 if (block->resized) {
2113 block->resized(block->idstr, newsize, block->host);
2114 }
2115 return 0;
2116}
2117
2118
2119static void dirty_memory_extend(ram_addr_t old_ram_size,
2120 ram_addr_t new_ram_size)
2121{
2122 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
2123 DIRTY_MEMORY_BLOCK_SIZE);
2124 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
2125 DIRTY_MEMORY_BLOCK_SIZE);
2126 int i;
2127
2128
2129 if (new_num_blocks <= old_num_blocks) {
2130 return;
2131 }
2132
2133 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
2134 DirtyMemoryBlocks *old_blocks;
2135 DirtyMemoryBlocks *new_blocks;
2136 int j;
2137
2138 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
2139 new_blocks = g_malloc(sizeof(*new_blocks) +
2140 sizeof(new_blocks->blocks[0]) * new_num_blocks);
2141
2142 if (old_num_blocks) {
2143 memcpy(new_blocks->blocks, old_blocks->blocks,
2144 old_num_blocks * sizeof(old_blocks->blocks[0]));
2145 }
2146
2147 for (j = old_num_blocks; j < new_num_blocks; j++) {
2148 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
2149 }
2150
2151 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
2152
2153 if (old_blocks) {
2154 g_free_rcu(old_blocks, rcu);
2155 }
2156 }
2157}
2158
2159static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)
2160{
2161 RAMBlock *block;
2162 RAMBlock *last_block = NULL;
2163 ram_addr_t old_ram_size, new_ram_size;
2164 Error *err = NULL;
2165
2166 old_ram_size = last_ram_page();
2167
2168 qemu_mutex_lock_ramlist();
2169 new_block->offset = find_ram_offset(new_block->max_length);
2170
2171 if (!new_block->host) {
2172 if (xen_enabled()) {
2173 xen_ram_alloc(new_block->offset, new_block->max_length,
2174 new_block->mr, &err);
2175 if (err) {
2176 error_propagate(errp, err);
2177 qemu_mutex_unlock_ramlist();
2178 return;
2179 }
2180 } else {
2181 new_block->host = phys_mem_alloc(new_block->max_length,
2182 &new_block->mr->align, shared);
2183 if (!new_block->host) {
2184 error_setg_errno(errp, errno,
2185 "cannot set up guest memory '%s'",
2186 memory_region_name(new_block->mr));
2187 qemu_mutex_unlock_ramlist();
2188 return;
2189 }
2190 memory_try_enable_merging(new_block->host, new_block->max_length);
2191 }
2192 }
2193
2194 new_ram_size = MAX(old_ram_size,
2195 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
2196 if (new_ram_size > old_ram_size) {
2197 dirty_memory_extend(old_ram_size, new_ram_size);
2198 }
2199
2200
2201
2202
2203 RAMBLOCK_FOREACH(block) {
2204 last_block = block;
2205 if (block->max_length < new_block->max_length) {
2206 break;
2207 }
2208 }
2209 if (block) {
2210 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
2211 } else if (last_block) {
2212 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
2213 } else {
2214 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
2215 }
2216 ram_list.mru_block = NULL;
2217
2218
2219 smp_wmb();
2220 ram_list.version++;
2221 qemu_mutex_unlock_ramlist();
2222
2223 cpu_physical_memory_set_dirty_range(new_block->offset,
2224 new_block->used_length,
2225 DIRTY_CLIENTS_ALL);
2226
2227 if (new_block->host) {
2228 qemu_ram_setup_dump(new_block->host, new_block->max_length);
2229 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
2230
2231 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
2232 ram_block_notify_add(new_block->host, new_block->max_length);
2233 }
2234}
2235
2236#ifdef CONFIG_POSIX
2237RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
2238 uint32_t ram_flags, int fd,
2239 Error **errp)
2240{
2241 RAMBlock *new_block;
2242 Error *local_err = NULL;
2243 int64_t file_size;
2244
2245
2246 assert((ram_flags & ~(RAM_SHARED | RAM_PMEM)) == 0);
2247
2248 if (xen_enabled()) {
2249 error_setg(errp, "-mem-path not supported with Xen");
2250 return NULL;
2251 }
2252
2253 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2254 error_setg(errp,
2255 "host lacks kvm mmu notifiers, -mem-path unsupported");
2256 return NULL;
2257 }
2258
2259 if (phys_mem_alloc != qemu_anon_ram_alloc) {
2260
2261
2262
2263
2264
2265 error_setg(errp,
2266 "-mem-path not supported with this accelerator");
2267 return NULL;
2268 }
2269
2270 size = HOST_PAGE_ALIGN(size);
2271 file_size = get_file_size(fd);
2272 if (file_size > 0 && file_size < size) {
2273 error_setg(errp, "backing store %s size 0x%" PRIx64
2274 " does not match 'size' option 0x" RAM_ADDR_FMT,
2275 mem_path, file_size, size);
2276 return NULL;
2277 }
2278
2279 new_block = g_malloc0(sizeof(*new_block));
2280 new_block->mr = mr;
2281 new_block->used_length = size;
2282 new_block->max_length = size;
2283 new_block->flags = ram_flags;
2284 new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
2285 if (!new_block->host) {
2286 g_free(new_block);
2287 return NULL;
2288 }
2289
2290 ram_block_add(new_block, &local_err, ram_flags & RAM_SHARED);
2291 if (local_err) {
2292 g_free(new_block);
2293 error_propagate(errp, local_err);
2294 return NULL;
2295 }
2296 return new_block;
2297
2298}
2299
2300
2301RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
2302 uint32_t ram_flags, const char *mem_path,
2303 Error **errp)
2304{
2305 int fd;
2306 bool created;
2307 RAMBlock *block;
2308
2309 fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
2310 if (fd < 0) {
2311 return NULL;
2312 }
2313
2314 block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, errp);
2315 if (!block) {
2316 if (created) {
2317 unlink(mem_path);
2318 }
2319 close(fd);
2320 return NULL;
2321 }
2322
2323 return block;
2324}
2325#endif
2326
2327static
2328RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2329 void (*resized)(const char*,
2330 uint64_t length,
2331 void *host),
2332 void *host, bool resizeable, bool share,
2333 MemoryRegion *mr, Error **errp)
2334{
2335 RAMBlock *new_block;
2336 Error *local_err = NULL;
2337
2338 size = HOST_PAGE_ALIGN(size);
2339 max_size = HOST_PAGE_ALIGN(max_size);
2340 new_block = g_malloc0(sizeof(*new_block));
2341 new_block->mr = mr;
2342 new_block->resized = resized;
2343 new_block->used_length = size;
2344 new_block->max_length = max_size;
2345 assert(max_size >= size);
2346 new_block->fd = -1;
2347 new_block->page_size = getpagesize();
2348 new_block->host = host;
2349 if (host) {
2350 new_block->flags |= RAM_PREALLOC;
2351 }
2352 if (resizeable) {
2353 new_block->flags |= RAM_RESIZEABLE;
2354 }
2355 ram_block_add(new_block, &local_err, share);
2356 if (local_err) {
2357 g_free(new_block);
2358 error_propagate(errp, local_err);
2359 return NULL;
2360 }
2361 return new_block;
2362}
2363
2364RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
2365 MemoryRegion *mr, Error **errp)
2366{
2367 return qemu_ram_alloc_internal(size, size, NULL, host, false,
2368 false, mr, errp);
2369}
2370
2371RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share,
2372 MemoryRegion *mr, Error **errp)
2373{
2374 return qemu_ram_alloc_internal(size, size, NULL, NULL, false,
2375 share, mr, errp);
2376}
2377
2378RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
2379 void (*resized)(const char*,
2380 uint64_t length,
2381 void *host),
2382 MemoryRegion *mr, Error **errp)
2383{
2384 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true,
2385 false, mr, errp);
2386}
2387
2388static void reclaim_ramblock(RAMBlock *block)
2389{
2390 if (block->flags & RAM_PREALLOC) {
2391 ;
2392 } else if (xen_enabled()) {
2393 xen_invalidate_map_cache_entry(block->host);
2394#ifndef _WIN32
2395 } else if (block->fd >= 0) {
2396 qemu_ram_munmap(block->host, block->max_length);
2397 close(block->fd);
2398#endif
2399 } else {
2400 qemu_anon_ram_free(block->host, block->max_length);
2401 }
2402 g_free(block);
2403}
2404
2405void qemu_ram_free(RAMBlock *block)
2406{
2407 if (!block) {
2408 return;
2409 }
2410
2411 if (block->host) {
2412 ram_block_notify_remove(block->host, block->max_length);
2413 }
2414
2415 qemu_mutex_lock_ramlist();
2416 QLIST_REMOVE_RCU(block, next);
2417 ram_list.mru_block = NULL;
2418
2419 smp_wmb();
2420 ram_list.version++;
2421 call_rcu(block, reclaim_ramblock, rcu);
2422 qemu_mutex_unlock_ramlist();
2423}
2424
2425#ifndef _WIN32
2426void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2427{
2428 RAMBlock *block;
2429 ram_addr_t offset;
2430 int flags;
2431 void *area, *vaddr;
2432
2433 RAMBLOCK_FOREACH(block) {
2434 offset = addr - block->offset;
2435 if (offset < block->max_length) {
2436 vaddr = ramblock_ptr(block, offset);
2437 if (block->flags & RAM_PREALLOC) {
2438 ;
2439 } else if (xen_enabled()) {
2440 abort();
2441 } else {
2442 flags = MAP_FIXED;
2443 if (block->fd >= 0) {
2444 flags |= (block->flags & RAM_SHARED ?
2445 MAP_SHARED : MAP_PRIVATE);
2446 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2447 flags, block->fd, offset);
2448 } else {
2449
2450
2451
2452
2453
2454 assert(phys_mem_alloc == qemu_anon_ram_alloc);
2455
2456 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
2457 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2458 flags, -1, 0);
2459 }
2460 if (area != vaddr) {
2461 error_report("Could not remap addr: "
2462 RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
2463 length, addr);
2464 exit(1);
2465 }
2466 memory_try_enable_merging(vaddr, length);
2467 qemu_ram_setup_dump(vaddr, length);
2468 }
2469 }
2470 }
2471}
2472#endif
2473
2474
2475
2476
2477
2478
2479
2480
2481void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
2482{
2483 RAMBlock *block = ram_block;
2484
2485 if (block == NULL) {
2486 block = qemu_get_ram_block(addr);
2487 addr -= block->offset;
2488 }
2489
2490 if (xen_enabled() && block->host == NULL) {
2491
2492
2493
2494
2495 if (block->offset == 0) {
2496 return xen_map_cache(addr, 0, 0, false);
2497 }
2498
2499 block->host = xen_map_cache(block->offset, block->max_length, 1, false);
2500 }
2501 return ramblock_ptr(block, addr);
2502}
2503
2504
2505
2506
2507
2508
2509static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
2510 hwaddr *size, bool lock)
2511{
2512 RAMBlock *block = ram_block;
2513 if (*size == 0) {
2514 return NULL;
2515 }
2516
2517 if (block == NULL) {
2518 block = qemu_get_ram_block(addr);
2519 addr -= block->offset;
2520 }
2521 *size = MIN(*size, block->max_length - addr);
2522
2523 if (xen_enabled() && block->host == NULL) {
2524
2525
2526
2527
2528 if (block->offset == 0) {
2529 return xen_map_cache(addr, *size, lock, lock);
2530 }
2531
2532 block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
2533 }
2534
2535 return ramblock_ptr(block, addr);
2536}
2537
2538
2539ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
2540{
2541 ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
2542 assert((uintptr_t)host >= (uintptr_t)rb->host);
2543 assert(res < rb->max_length);
2544
2545 return res;
2546}
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
2566 ram_addr_t *offset)
2567{
2568 RAMBlock *block;
2569 uint8_t *host = ptr;
2570
2571 if (xen_enabled()) {
2572 ram_addr_t ram_addr;
2573 rcu_read_lock();
2574 ram_addr = xen_ram_addr_from_mapcache(ptr);
2575 block = qemu_get_ram_block(ram_addr);
2576 if (block) {
2577 *offset = ram_addr - block->offset;
2578 }
2579 rcu_read_unlock();
2580 return block;
2581 }
2582
2583 rcu_read_lock();
2584 block = atomic_rcu_read(&ram_list.mru_block);
2585 if (block && block->host && host - block->host < block->max_length) {
2586 goto found;
2587 }
2588
2589 RAMBLOCK_FOREACH(block) {
2590
2591 if (block->host == NULL) {
2592 continue;
2593 }
2594 if (host - block->host < block->max_length) {
2595 goto found;
2596 }
2597 }
2598
2599 rcu_read_unlock();
2600 return NULL;
2601
2602found:
2603 *offset = (host - block->host);
2604 if (round_offset) {
2605 *offset &= TARGET_PAGE_MASK;
2606 }
2607 rcu_read_unlock();
2608 return block;
2609}
2610
2611
2612
2613
2614
2615
2616
2617
2618RAMBlock *qemu_ram_block_by_name(const char *name)
2619{
2620 RAMBlock *block;
2621
2622 RAMBLOCK_FOREACH(block) {
2623 if (!strcmp(name, block->idstr)) {
2624 return block;
2625 }
2626 }
2627
2628 return NULL;
2629}
2630
2631
2632
2633ram_addr_t qemu_ram_addr_from_host(void *ptr)
2634{
2635 RAMBlock *block;
2636 ram_addr_t offset;
2637
2638 block = qemu_ram_block_from_host(ptr, false, &offset);
2639 if (!block) {
2640 return RAM_ADDR_INVALID;
2641 }
2642
2643 return block->offset + offset;
2644}
2645
2646
2647void memory_notdirty_write_prepare(NotDirtyInfo *ndi,
2648 CPUState *cpu,
2649 vaddr mem_vaddr,
2650 ram_addr_t ram_addr,
2651 unsigned size)
2652{
2653 ndi->cpu = cpu;
2654 ndi->ram_addr = ram_addr;
2655 ndi->mem_vaddr = mem_vaddr;
2656 ndi->size = size;
2657 ndi->pages = NULL;
2658
2659 assert(tcg_enabled());
2660 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
2661 ndi->pages = page_collection_lock(ram_addr, ram_addr + size);
2662 tb_invalidate_phys_page_fast(ndi->pages, ram_addr, size);
2663 }
2664}
2665
2666
2667void memory_notdirty_write_complete(NotDirtyInfo *ndi)
2668{
2669 if (ndi->pages) {
2670 assert(tcg_enabled());
2671 page_collection_unlock(ndi->pages);
2672 ndi->pages = NULL;
2673 }
2674
2675
2676
2677
2678 cpu_physical_memory_set_dirty_range(ndi->ram_addr, ndi->size,
2679 DIRTY_CLIENTS_NOCODE);
2680
2681
2682 if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {
2683 tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);
2684 }
2685}
2686
2687
2688static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
2689 uint64_t val, unsigned size)
2690{
2691 NotDirtyInfo ndi;
2692
2693 memory_notdirty_write_prepare(&ndi, current_cpu, current_cpu->mem_io_vaddr,
2694 ram_addr, size);
2695
2696 stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
2697 memory_notdirty_write_complete(&ndi);
2698}
2699
2700static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
2701 unsigned size, bool is_write,
2702 MemTxAttrs attrs)
2703{
2704 return is_write;
2705}
2706
2707static const MemoryRegionOps notdirty_mem_ops = {
2708 .write = notdirty_mem_write,
2709 .valid.accepts = notdirty_mem_accepts,
2710 .endianness = DEVICE_NATIVE_ENDIAN,
2711 .valid = {
2712 .min_access_size = 1,
2713 .max_access_size = 8,
2714 .unaligned = false,
2715 },
2716 .impl = {
2717 .min_access_size = 1,
2718 .max_access_size = 8,
2719 .unaligned = false,
2720 },
2721};
2722
2723
2724static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
2725{
2726 CPUState *cpu = current_cpu;
2727 CPUClass *cc = CPU_GET_CLASS(cpu);
2728 target_ulong vaddr;
2729 CPUWatchpoint *wp;
2730
2731 assert(tcg_enabled());
2732 if (cpu->watchpoint_hit) {
2733
2734
2735
2736 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
2737 return;
2738 }
2739 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2740 vaddr = cc->adjust_watchpoint_address(cpu, vaddr, len);
2741 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
2742 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2743 && (wp->flags & flags)) {
2744 if (flags == BP_MEM_READ) {
2745 wp->flags |= BP_WATCHPOINT_HIT_READ;
2746 } else {
2747 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2748 }
2749 wp->hitaddr = vaddr;
2750 wp->hitattrs = attrs;
2751 if (!cpu->watchpoint_hit) {
2752 if (wp->flags & BP_CPU &&
2753 !cc->debug_check_watchpoint(cpu, wp)) {
2754 wp->flags &= ~BP_WATCHPOINT_HIT;
2755 continue;
2756 }
2757 cpu->watchpoint_hit = wp;
2758
2759 mmap_lock();
2760 tb_check_watchpoint(cpu);
2761 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2762 cpu->exception_index = EXCP_DEBUG;
2763 mmap_unlock();
2764 cpu_loop_exit(cpu);
2765 } else {
2766
2767 cpu->cflags_next_tb = 1 | curr_cflags();
2768 mmap_unlock();
2769 cpu_loop_exit_noexc(cpu);
2770 }
2771 }
2772 } else {
2773 wp->flags &= ~BP_WATCHPOINT_HIT;
2774 }
2775 }
2776}
2777
2778
2779
2780
2781static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2782 unsigned size, MemTxAttrs attrs)
2783{
2784 MemTxResult res;
2785 uint64_t data;
2786 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2787 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2788
2789 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2790 switch (size) {
2791 case 1:
2792 data = address_space_ldub(as, addr, attrs, &res);
2793 break;
2794 case 2:
2795 data = address_space_lduw(as, addr, attrs, &res);
2796 break;
2797 case 4:
2798 data = address_space_ldl(as, addr, attrs, &res);
2799 break;
2800 case 8:
2801 data = address_space_ldq(as, addr, attrs, &res);
2802 break;
2803 default: abort();
2804 }
2805 *pdata = data;
2806 return res;
2807}
2808
2809static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2810 uint64_t val, unsigned size,
2811 MemTxAttrs attrs)
2812{
2813 MemTxResult res;
2814 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2815 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2816
2817 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2818 switch (size) {
2819 case 1:
2820 address_space_stb(as, addr, val, attrs, &res);
2821 break;
2822 case 2:
2823 address_space_stw(as, addr, val, attrs, &res);
2824 break;
2825 case 4:
2826 address_space_stl(as, addr, val, attrs, &res);
2827 break;
2828 case 8:
2829 address_space_stq(as, addr, val, attrs, &res);
2830 break;
2831 default: abort();
2832 }
2833 return res;
2834}
2835
2836static const MemoryRegionOps watch_mem_ops = {
2837 .read_with_attrs = watch_mem_read,
2838 .write_with_attrs = watch_mem_write,
2839 .endianness = DEVICE_NATIVE_ENDIAN,
2840 .valid = {
2841 .min_access_size = 1,
2842 .max_access_size = 8,
2843 .unaligned = false,
2844 },
2845 .impl = {
2846 .min_access_size = 1,
2847 .max_access_size = 8,
2848 .unaligned = false,
2849 },
2850};
2851
2852static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
2853 MemTxAttrs attrs, uint8_t *buf, int len);
2854static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
2855 const uint8_t *buf, int len);
2856static bool flatview_access_valid(FlatView *fv, hwaddr addr, int len,
2857 bool is_write, MemTxAttrs attrs);
2858
2859static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2860 unsigned len, MemTxAttrs attrs)
2861{
2862 subpage_t *subpage = opaque;
2863 uint8_t buf[8];
2864 MemTxResult res;
2865
2866#if defined(DEBUG_SUBPAGE)
2867 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2868 subpage, len, addr);
2869#endif
2870 res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
2871 if (res) {
2872 return res;
2873 }
2874 *data = ldn_p(buf, len);
2875 return MEMTX_OK;
2876}
2877
2878static MemTxResult subpage_write(void *opaque, hwaddr addr,
2879 uint64_t value, unsigned len, MemTxAttrs attrs)
2880{
2881 subpage_t *subpage = opaque;
2882 uint8_t buf[8];
2883
2884#if defined(DEBUG_SUBPAGE)
2885 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2886 " value %"PRIx64"\n",
2887 __func__, subpage, len, addr, value);
2888#endif
2889 stn_p(buf, len, value);
2890 return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
2891}
2892
2893static bool subpage_accepts(void *opaque, hwaddr addr,
2894 unsigned len, bool is_write,
2895 MemTxAttrs attrs)
2896{
2897 subpage_t *subpage = opaque;
2898#if defined(DEBUG_SUBPAGE)
2899 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2900 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2901#endif
2902
2903 return flatview_access_valid(subpage->fv, addr + subpage->base,
2904 len, is_write, attrs);
2905}
2906
2907static const MemoryRegionOps subpage_ops = {
2908 .read_with_attrs = subpage_read,
2909 .write_with_attrs = subpage_write,
2910 .impl.min_access_size = 1,
2911 .impl.max_access_size = 8,
2912 .valid.min_access_size = 1,
2913 .valid.max_access_size = 8,
2914 .valid.accepts = subpage_accepts,
2915 .endianness = DEVICE_NATIVE_ENDIAN,
2916};
2917
2918static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2919 uint16_t section)
2920{
2921 int idx, eidx;
2922
2923 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2924 return -1;
2925 idx = SUBPAGE_IDX(start);
2926 eidx = SUBPAGE_IDX(end);
2927#if defined(DEBUG_SUBPAGE)
2928 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2929 __func__, mmio, start, end, idx, eidx, section);
2930#endif
2931 for (; idx <= eidx; idx++) {
2932 mmio->sub_section[idx] = section;
2933 }
2934
2935 return 0;
2936}
2937
2938static subpage_t *subpage_init(FlatView *fv, hwaddr base)
2939{
2940 subpage_t *mmio;
2941
2942 mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
2943 mmio->fv = fv;
2944 mmio->base = base;
2945 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2946 NULL, TARGET_PAGE_SIZE);
2947 mmio->iomem.subpage = true;
2948#if defined(DEBUG_SUBPAGE)
2949 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2950 mmio, base, TARGET_PAGE_SIZE);
2951#endif
2952 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2953
2954 return mmio;
2955}
2956
2957static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
2958{
2959 assert(fv);
2960 MemoryRegionSection section = {
2961 .fv = fv,
2962 .mr = mr,
2963 .offset_within_address_space = 0,
2964 .offset_within_region = 0,
2965 .size = int128_2_64(),
2966 };
2967
2968 return phys_section_add(map, §ion);
2969}
2970
2971static void readonly_mem_write(void *opaque, hwaddr addr,
2972 uint64_t val, unsigned size)
2973{
2974
2975}
2976
2977static bool readonly_mem_accepts(void *opaque, hwaddr addr,
2978 unsigned size, bool is_write,
2979 MemTxAttrs attrs)
2980{
2981 return is_write;
2982}
2983
2984
2985
2986
2987static const MemoryRegionOps readonly_mem_ops = {
2988 .write = readonly_mem_write,
2989 .valid.accepts = readonly_mem_accepts,
2990 .endianness = DEVICE_NATIVE_ENDIAN,
2991 .valid = {
2992 .min_access_size = 1,
2993 .max_access_size = 8,
2994 .unaligned = false,
2995 },
2996 .impl = {
2997 .min_access_size = 1,
2998 .max_access_size = 8,
2999 .unaligned = false,
3000 },
3001};
3002
3003MemoryRegionSection *iotlb_to_section(CPUState *cpu,
3004 hwaddr index, MemTxAttrs attrs)
3005{
3006 int asidx = cpu_asidx_from_attrs(cpu, attrs);
3007 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
3008 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
3009 MemoryRegionSection *sections = d->map.sections;
3010
3011 return §ions[index & ~TARGET_PAGE_MASK];
3012}
3013
3014static void io_mem_init(void)
3015{
3016 memory_region_init_io(&io_mem_rom, NULL, &readonly_mem_ops,
3017 NULL, NULL, UINT64_MAX);
3018 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
3019 NULL, UINT64_MAX);
3020
3021
3022
3023
3024 memory_region_init_io(&io_mem_notdirty, NULL, ¬dirty_mem_ops, NULL,
3025 NULL, UINT64_MAX);
3026 memory_region_clear_global_locking(&io_mem_notdirty);
3027
3028 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
3029 NULL, UINT64_MAX);
3030}
3031
3032AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
3033{
3034 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
3035 uint16_t n;
3036
3037 n = dummy_section(&d->map, fv, &io_mem_unassigned);
3038 assert(n == PHYS_SECTION_UNASSIGNED);
3039 n = dummy_section(&d->map, fv, &io_mem_notdirty);
3040 assert(n == PHYS_SECTION_NOTDIRTY);
3041 n = dummy_section(&d->map, fv, &io_mem_rom);
3042 assert(n == PHYS_SECTION_ROM);
3043 n = dummy_section(&d->map, fv, &io_mem_watch);
3044 assert(n == PHYS_SECTION_WATCH);
3045
3046 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
3047
3048 return d;
3049}
3050
3051void address_space_dispatch_free(AddressSpaceDispatch *d)
3052{
3053 phys_sections_free(&d->map);
3054 g_free(d);
3055}
3056
3057static void tcg_commit(MemoryListener *listener)
3058{
3059 CPUAddressSpace *cpuas;
3060 AddressSpaceDispatch *d;
3061
3062 assert(tcg_enabled());
3063
3064
3065 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
3066 cpu_reloading_memory_map();
3067
3068
3069
3070
3071 d = address_space_to_dispatch(cpuas->as);
3072 atomic_rcu_set(&cpuas->memory_dispatch, d);
3073 tlb_flush(cpuas->cpu);
3074}
3075
3076static void memory_map_init(void)
3077{
3078 system_memory = g_malloc(sizeof(*system_memory));
3079
3080 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
3081 address_space_init(&address_space_memory, system_memory, "memory");
3082
3083 system_io = g_malloc(sizeof(*system_io));
3084 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
3085 65536);
3086 address_space_init(&address_space_io, system_io, "I/O");
3087}
3088
3089MemoryRegion *get_system_memory(void)
3090{
3091 return system_memory;
3092}
3093
3094MemoryRegion *get_system_io(void)
3095{
3096 return system_io;
3097}
3098
3099#endif
3100
3101
3102#if defined(CONFIG_USER_ONLY)
3103int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3104 uint8_t *buf, int len, int is_write)
3105{
3106 int l, flags;
3107 target_ulong page;
3108 void * p;
3109
3110 while (len > 0) {
3111 page = addr & TARGET_PAGE_MASK;
3112 l = (page + TARGET_PAGE_SIZE) - addr;
3113 if (l > len)
3114 l = len;
3115 flags = page_get_flags(page);
3116 if (!(flags & PAGE_VALID))
3117 return -1;
3118 if (is_write) {
3119 if (!(flags & PAGE_WRITE))
3120 return -1;
3121
3122 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3123 return -1;
3124 memcpy(p, buf, l);
3125 unlock_user(p, addr, l);
3126 } else {
3127 if (!(flags & PAGE_READ))
3128 return -1;
3129
3130 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3131 return -1;
3132 memcpy(buf, p, l);
3133 unlock_user(p, addr, 0);
3134 }
3135 len -= l;
3136 buf += l;
3137 addr += l;
3138 }
3139 return 0;
3140}
3141
3142#else
3143
3144static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
3145 hwaddr length)
3146{
3147 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3148 addr += memory_region_get_ram_addr(mr);
3149
3150
3151
3152
3153
3154 if (dirty_log_mask) {
3155 dirty_log_mask =
3156 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
3157 }
3158 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
3159 assert(tcg_enabled());
3160 tb_invalidate_phys_range(addr, addr + length);
3161 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3162 }
3163 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
3164}
3165
3166static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
3167{
3168 unsigned access_size_max = mr->ops->valid.max_access_size;
3169
3170
3171
3172 if (access_size_max == 0) {
3173 access_size_max = 4;
3174 }
3175
3176
3177 if (!mr->ops->impl.unaligned) {
3178 unsigned align_size_max = addr & -addr;
3179 if (align_size_max != 0 && align_size_max < access_size_max) {
3180 access_size_max = align_size_max;
3181 }
3182 }
3183
3184
3185 if (l > access_size_max) {
3186 l = access_size_max;
3187 }
3188 l = pow2floor(l);
3189
3190 return l;
3191}
3192
3193static bool prepare_mmio_access(MemoryRegion *mr)
3194{
3195 bool unlocked = !qemu_mutex_iothread_locked();
3196 bool release_lock = false;
3197
3198 if (unlocked && mr->global_locking) {
3199 qemu_mutex_lock_iothread();
3200 unlocked = false;
3201 release_lock = true;
3202 }
3203 if (mr->flush_coalesced_mmio) {
3204 if (unlocked) {
3205 qemu_mutex_lock_iothread();
3206 }
3207 qemu_flush_coalesced_mmio_buffer();
3208 if (unlocked) {
3209 qemu_mutex_unlock_iothread();
3210 }
3211 }
3212
3213 return release_lock;
3214}
3215
3216
3217static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
3218 MemTxAttrs attrs,
3219 const uint8_t *buf,
3220 int len, hwaddr addr1,
3221 hwaddr l, MemoryRegion *mr)
3222{
3223 uint8_t *ptr;
3224 uint64_t val;
3225 MemTxResult result = MEMTX_OK;
3226 bool release_lock = false;
3227
3228 for (;;) {
3229 if (!memory_access_is_direct(mr, true)) {
3230 release_lock |= prepare_mmio_access(mr);
3231 l = memory_access_size(mr, l, addr1);
3232
3233
3234 val = ldn_p(buf, l);
3235 result |= memory_region_dispatch_write(mr, addr1, val, l, attrs);
3236 } else {
3237
3238 ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
3239 memcpy(ptr, buf, l);
3240 invalidate_and_set_dirty(mr, addr1, l);
3241 }
3242
3243 if (release_lock) {
3244 qemu_mutex_unlock_iothread();
3245 release_lock = false;
3246 }
3247
3248 len -= l;
3249 buf += l;
3250 addr += l;
3251
3252 if (!len) {
3253 break;
3254 }
3255
3256 l = len;
3257 mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
3258 }
3259
3260 return result;
3261}
3262
3263
3264static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
3265 const uint8_t *buf, int len)
3266{
3267 hwaddr l;
3268 hwaddr addr1;
3269 MemoryRegion *mr;
3270 MemTxResult result = MEMTX_OK;
3271
3272 l = len;
3273 mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
3274 result = flatview_write_continue(fv, addr, attrs, buf, len,
3275 addr1, l, mr);
3276
3277 return result;
3278}
3279
3280
3281MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
3282 MemTxAttrs attrs, uint8_t *buf,
3283 int len, hwaddr addr1, hwaddr l,
3284 MemoryRegion *mr)
3285{
3286 uint8_t *ptr;
3287 uint64_t val;
3288 MemTxResult result = MEMTX_OK;
3289 bool release_lock = false;
3290
3291 for (;;) {
3292 if (!memory_access_is_direct(mr, false)) {
3293
3294 release_lock |= prepare_mmio_access(mr);
3295 l = memory_access_size(mr, l, addr1);
3296 result |= memory_region_dispatch_read(mr, addr1, &val, l, attrs);
3297 stn_p(buf, l, val);
3298 } else {
3299
3300 ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
3301 memcpy(buf, ptr, l);
3302 }
3303
3304 if (release_lock) {
3305 qemu_mutex_unlock_iothread();
3306 release_lock = false;
3307 }
3308
3309 len -= l;
3310 buf += l;
3311 addr += l;
3312
3313 if (!len) {
3314 break;
3315 }
3316
3317 l = len;
3318 mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
3319 }
3320
3321 return result;
3322}
3323
3324
3325static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
3326 MemTxAttrs attrs, uint8_t *buf, int len)
3327{
3328 hwaddr l;
3329 hwaddr addr1;
3330 MemoryRegion *mr;
3331
3332 l = len;
3333 mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
3334 return flatview_read_continue(fv, addr, attrs, buf, len,
3335 addr1, l, mr);
3336}
3337
3338MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
3339 MemTxAttrs attrs, uint8_t *buf, int len)
3340{
3341 MemTxResult result = MEMTX_OK;
3342 FlatView *fv;
3343
3344 if (len > 0) {
3345 rcu_read_lock();
3346 fv = address_space_to_flatview(as);
3347 result = flatview_read(fv, addr, attrs, buf, len);
3348 rcu_read_unlock();
3349 }
3350
3351 return result;
3352}
3353
3354MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
3355 MemTxAttrs attrs,
3356 const uint8_t *buf, int len)
3357{
3358 MemTxResult result = MEMTX_OK;
3359 FlatView *fv;
3360
3361 if (len > 0) {
3362 rcu_read_lock();
3363 fv = address_space_to_flatview(as);
3364 result = flatview_write(fv, addr, attrs, buf, len);
3365 rcu_read_unlock();
3366 }
3367
3368 return result;
3369}
3370
3371MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
3372 uint8_t *buf, int len, bool is_write)
3373{
3374 if (is_write) {
3375 return address_space_write(as, addr, attrs, buf, len);
3376 } else {
3377 return address_space_read_full(as, addr, attrs, buf, len);
3378 }
3379}
3380
3381void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
3382 int len, int is_write)
3383{
3384 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
3385 buf, len, is_write);
3386}
3387
3388enum write_rom_type {
3389 WRITE_DATA,
3390 FLUSH_CACHE,
3391};
3392
3393static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
3394 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
3395{
3396 hwaddr l;
3397 uint8_t *ptr;
3398 hwaddr addr1;
3399 MemoryRegion *mr;
3400
3401 rcu_read_lock();
3402 while (len > 0) {
3403 l = len;
3404 mr = address_space_translate(as, addr, &addr1, &l, true,
3405 MEMTXATTRS_UNSPECIFIED);
3406
3407 if (!(memory_region_is_ram(mr) ||
3408 memory_region_is_romd(mr))) {
3409 l = memory_access_size(mr, l, addr1);
3410 } else {
3411
3412 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3413 switch (type) {
3414 case WRITE_DATA:
3415 memcpy(ptr, buf, l);
3416 invalidate_and_set_dirty(mr, addr1, l);
3417 break;
3418 case FLUSH_CACHE:
3419 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
3420 break;
3421 }
3422 }
3423 len -= l;
3424 buf += l;
3425 addr += l;
3426 }
3427 rcu_read_unlock();
3428}
3429
3430
3431void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
3432 const uint8_t *buf, int len)
3433{
3434 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
3435}
3436
3437void cpu_flush_icache_range(hwaddr start, int len)
3438{
3439
3440
3441
3442
3443
3444
3445 if (tcg_enabled()) {
3446 return;
3447 }
3448
3449 cpu_physical_memory_write_rom_internal(&address_space_memory,
3450 start, NULL, len, FLUSH_CACHE);
3451}
3452
3453typedef struct {
3454 MemoryRegion *mr;
3455 void *buffer;
3456 hwaddr addr;
3457 hwaddr len;
3458 bool in_use;
3459} BounceBuffer;
3460
3461static BounceBuffer bounce;
3462
3463typedef struct MapClient {
3464 QEMUBH *bh;
3465 QLIST_ENTRY(MapClient) link;
3466} MapClient;
3467
3468QemuMutex map_client_list_lock;
3469static QLIST_HEAD(map_client_list, MapClient) map_client_list
3470 = QLIST_HEAD_INITIALIZER(map_client_list);
3471
3472static void cpu_unregister_map_client_do(MapClient *client)
3473{
3474 QLIST_REMOVE(client, link);
3475 g_free(client);
3476}
3477
3478static void cpu_notify_map_clients_locked(void)
3479{
3480 MapClient *client;
3481
3482 while (!QLIST_EMPTY(&map_client_list)) {
3483 client = QLIST_FIRST(&map_client_list);
3484 qemu_bh_schedule(client->bh);
3485 cpu_unregister_map_client_do(client);
3486 }
3487}
3488
3489void cpu_register_map_client(QEMUBH *bh)
3490{
3491 MapClient *client = g_malloc(sizeof(*client));
3492
3493 qemu_mutex_lock(&map_client_list_lock);
3494 client->bh = bh;
3495 QLIST_INSERT_HEAD(&map_client_list, client, link);
3496 if (!atomic_read(&bounce.in_use)) {
3497 cpu_notify_map_clients_locked();
3498 }
3499 qemu_mutex_unlock(&map_client_list_lock);
3500}
3501
3502void cpu_exec_init_all(void)
3503{
3504 qemu_mutex_init(&ram_list.mutex);
3505
3506
3507
3508
3509
3510
3511
3512 finalize_target_page_bits();
3513 io_mem_init();
3514 memory_map_init();
3515 qemu_mutex_init(&map_client_list_lock);
3516}
3517
3518void cpu_unregister_map_client(QEMUBH *bh)
3519{
3520 MapClient *client;
3521
3522 qemu_mutex_lock(&map_client_list_lock);
3523 QLIST_FOREACH(client, &map_client_list, link) {
3524 if (client->bh == bh) {
3525 cpu_unregister_map_client_do(client);
3526 break;
3527 }
3528 }
3529 qemu_mutex_unlock(&map_client_list_lock);
3530}
3531
3532static void cpu_notify_map_clients(void)
3533{
3534 qemu_mutex_lock(&map_client_list_lock);
3535 cpu_notify_map_clients_locked();
3536 qemu_mutex_unlock(&map_client_list_lock);
3537}
3538
3539static bool flatview_access_valid(FlatView *fv, hwaddr addr, int len,
3540 bool is_write, MemTxAttrs attrs)
3541{
3542 MemoryRegion *mr;
3543 hwaddr l, xlat;
3544
3545 while (len > 0) {
3546 l = len;
3547 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3548 if (!memory_access_is_direct(mr, is_write)) {
3549 l = memory_access_size(mr, l, addr);
3550 if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
3551 return false;
3552 }
3553 }
3554
3555 len -= l;
3556 addr += l;
3557 }
3558 return true;
3559}
3560
3561bool address_space_access_valid(AddressSpace *as, hwaddr addr,
3562 int len, bool is_write,
3563 MemTxAttrs attrs)
3564{
3565 FlatView *fv;
3566 bool result;
3567
3568 rcu_read_lock();
3569 fv = address_space_to_flatview(as);
3570 result = flatview_access_valid(fv, addr, len, is_write, attrs);
3571 rcu_read_unlock();
3572 return result;
3573}
3574
3575static hwaddr
3576flatview_extend_translation(FlatView *fv, hwaddr addr,
3577 hwaddr target_len,
3578 MemoryRegion *mr, hwaddr base, hwaddr len,
3579 bool is_write, MemTxAttrs attrs)
3580{
3581 hwaddr done = 0;
3582 hwaddr xlat;
3583 MemoryRegion *this_mr;
3584
3585 for (;;) {
3586 target_len -= len;
3587 addr += len;
3588 done += len;
3589 if (target_len == 0) {
3590 return done;
3591 }
3592
3593 len = target_len;
3594 this_mr = flatview_translate(fv, addr, &xlat,
3595 &len, is_write, attrs);
3596 if (this_mr != mr || xlat != base + done) {
3597 return done;
3598 }
3599 }
3600}
3601
3602
3603
3604
3605
3606
3607
3608
3609void *address_space_map(AddressSpace *as,
3610 hwaddr addr,
3611 hwaddr *plen,
3612 bool is_write,
3613 MemTxAttrs attrs)
3614{
3615 hwaddr len = *plen;
3616 hwaddr l, xlat;
3617 MemoryRegion *mr;
3618 void *ptr;
3619 FlatView *fv;
3620
3621 if (len == 0) {
3622 return NULL;
3623 }
3624
3625 l = len;
3626 rcu_read_lock();
3627 fv = address_space_to_flatview(as);
3628 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3629
3630 if (!memory_access_is_direct(mr, is_write)) {
3631 if (atomic_xchg(&bounce.in_use, true)) {
3632 rcu_read_unlock();
3633 return NULL;
3634 }
3635
3636 l = MIN(l, TARGET_PAGE_SIZE);
3637 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
3638 bounce.addr = addr;
3639 bounce.len = l;
3640
3641 memory_region_ref(mr);
3642 bounce.mr = mr;
3643 if (!is_write) {
3644 flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
3645 bounce.buffer, l);
3646 }
3647
3648 rcu_read_unlock();
3649 *plen = l;
3650 return bounce.buffer;
3651 }
3652
3653
3654 memory_region_ref(mr);
3655 *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
3656 l, is_write, attrs);
3657 ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
3658 rcu_read_unlock();
3659
3660 return ptr;
3661}
3662
3663
3664
3665
3666
3667void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3668 int is_write, hwaddr access_len)
3669{
3670 if (buffer != bounce.buffer) {
3671 MemoryRegion *mr;
3672 ram_addr_t addr1;
3673
3674 mr = memory_region_from_host(buffer, &addr1);
3675 assert(mr != NULL);
3676 if (is_write) {
3677 invalidate_and_set_dirty(mr, addr1, access_len);
3678 }
3679 if (xen_enabled()) {
3680 xen_invalidate_map_cache_entry(buffer);
3681 }
3682 memory_region_unref(mr);
3683 return;
3684 }
3685 if (is_write) {
3686 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
3687 bounce.buffer, access_len);
3688 }
3689 qemu_vfree(bounce.buffer);
3690 bounce.buffer = NULL;
3691 memory_region_unref(bounce.mr);
3692 atomic_mb_set(&bounce.in_use, false);
3693 cpu_notify_map_clients();
3694}
3695
3696void *cpu_physical_memory_map(hwaddr addr,
3697 hwaddr *plen,
3698 int is_write)
3699{
3700 return address_space_map(&address_space_memory, addr, plen, is_write,
3701 MEMTXATTRS_UNSPECIFIED);
3702}
3703
3704void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3705 int is_write, hwaddr access_len)
3706{
3707 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3708}
3709
3710#define ARG1_DECL AddressSpace *as
3711#define ARG1 as
3712#define SUFFIX
3713#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
3714#define RCU_READ_LOCK(...) rcu_read_lock()
3715#define RCU_READ_UNLOCK(...) rcu_read_unlock()
3716#include "memory_ldst.inc.c"
3717
3718int64_t address_space_cache_init(MemoryRegionCache *cache,
3719 AddressSpace *as,
3720 hwaddr addr,
3721 hwaddr len,
3722 bool is_write)
3723{
3724 AddressSpaceDispatch *d;
3725 hwaddr l;
3726 MemoryRegion *mr;
3727
3728 assert(len > 0);
3729
3730 l = len;
3731 cache->fv = address_space_get_flatview(as);
3732 d = flatview_to_dispatch(cache->fv);
3733 cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3734
3735 mr = cache->mrs.mr;
3736 memory_region_ref(mr);
3737 if (memory_access_is_direct(mr, is_write)) {
3738
3739
3740
3741
3742 l = flatview_extend_translation(cache->fv, addr, len, mr,
3743 cache->xlat, l, is_write,
3744 MEMTXATTRS_UNSPECIFIED);
3745 cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
3746 } else {
3747 cache->ptr = NULL;
3748 }
3749
3750 cache->len = l;
3751 cache->is_write = is_write;
3752 return l;
3753}
3754
3755void address_space_cache_invalidate(MemoryRegionCache *cache,
3756 hwaddr addr,
3757 hwaddr access_len)
3758{
3759 assert(cache->is_write);
3760 if (likely(cache->ptr)) {
3761 invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3762 }
3763}
3764
3765void address_space_cache_destroy(MemoryRegionCache *cache)
3766{
3767 if (!cache->mrs.mr) {
3768 return;
3769 }
3770
3771 if (xen_enabled()) {
3772 xen_invalidate_map_cache_entry(cache->ptr);
3773 }
3774 memory_region_unref(cache->mrs.mr);
3775 flatview_unref(cache->fv);
3776 cache->mrs.mr = NULL;
3777 cache->fv = NULL;
3778}
3779
3780
3781
3782
3783
3784
3785static inline MemoryRegion *address_space_translate_cached(
3786 MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
3787 hwaddr *plen, bool is_write, MemTxAttrs attrs)
3788{
3789 MemoryRegionSection section;
3790 MemoryRegion *mr;
3791 IOMMUMemoryRegion *iommu_mr;
3792 AddressSpace *target_as;
3793
3794 assert(!cache->ptr);
3795 *xlat = addr + cache->xlat;
3796
3797 mr = cache->mrs.mr;
3798 iommu_mr = memory_region_get_iommu(mr);
3799 if (!iommu_mr) {
3800
3801 return mr;
3802 }
3803
3804 section = address_space_translate_iommu(iommu_mr, xlat, plen,
3805 NULL, is_write, true,
3806 &target_as, attrs);
3807 return section.mr;
3808}
3809
3810
3811
3812
3813void
3814address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3815 void *buf, int len)
3816{
3817 hwaddr addr1, l;
3818 MemoryRegion *mr;
3819
3820 l = len;
3821 mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
3822 MEMTXATTRS_UNSPECIFIED);
3823 flatview_read_continue(cache->fv,
3824 addr, MEMTXATTRS_UNSPECIFIED, buf, len,
3825 addr1, l, mr);
3826}
3827
3828
3829
3830
3831void
3832address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3833 const void *buf, int len)
3834{
3835 hwaddr addr1, l;
3836 MemoryRegion *mr;
3837
3838 l = len;
3839 mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
3840 MEMTXATTRS_UNSPECIFIED);
3841 flatview_write_continue(cache->fv,
3842 addr, MEMTXATTRS_UNSPECIFIED, buf, len,
3843 addr1, l, mr);
3844}
3845
3846#define ARG1_DECL MemoryRegionCache *cache
3847#define ARG1 cache
3848#define SUFFIX _cached_slow
3849#define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
3850#define RCU_READ_LOCK() ((void)0)
3851#define RCU_READ_UNLOCK() ((void)0)
3852#include "memory_ldst.inc.c"
3853
3854
3855int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3856 uint8_t *buf, int len, int is_write)
3857{
3858 int l;
3859 hwaddr phys_addr;
3860 target_ulong page;
3861
3862 cpu_synchronize_state(cpu);
3863 while (len > 0) {
3864 int asidx;
3865 MemTxAttrs attrs;
3866
3867 page = addr & TARGET_PAGE_MASK;
3868 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3869 asidx = cpu_asidx_from_attrs(cpu, attrs);
3870
3871 if (phys_addr == -1)
3872 return -1;
3873 l = (page + TARGET_PAGE_SIZE) - addr;
3874 if (l > len)
3875 l = len;
3876 phys_addr += (addr & ~TARGET_PAGE_MASK);
3877 if (is_write) {
3878 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3879 phys_addr, buf, l);
3880 } else {
3881 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3882 MEMTXATTRS_UNSPECIFIED,
3883 buf, l, 0);
3884 }
3885 len -= l;
3886 buf += l;
3887 addr += l;
3888 }
3889 return 0;
3890}
3891
3892
3893
3894
3895
3896size_t qemu_target_page_size(void)
3897{
3898 return TARGET_PAGE_SIZE;
3899}
3900
3901int qemu_target_page_bits(void)
3902{
3903 return TARGET_PAGE_BITS;
3904}
3905
3906int qemu_target_page_bits_min(void)
3907{
3908 return TARGET_PAGE_BITS_MIN;
3909}
3910#endif
3911
3912bool target_words_bigendian(void)
3913{
3914#if defined(TARGET_WORDS_BIGENDIAN)
3915 return true;
3916#else
3917 return false;
3918#endif
3919}
3920
3921#ifndef CONFIG_USER_ONLY
3922bool cpu_physical_memory_is_io(hwaddr phys_addr)
3923{
3924 MemoryRegion*mr;
3925 hwaddr l = 1;
3926 bool res;
3927
3928 rcu_read_lock();
3929 mr = address_space_translate(&address_space_memory,
3930 phys_addr, &phys_addr, &l, false,
3931 MEMTXATTRS_UNSPECIFIED);
3932
3933 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3934 rcu_read_unlock();
3935 return res;
3936}
3937
3938int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3939{
3940 RAMBlock *block;
3941 int ret = 0;
3942
3943 rcu_read_lock();
3944 RAMBLOCK_FOREACH(block) {
3945 ret = func(block->idstr, block->host, block->offset,
3946 block->used_length, opaque);
3947 if (ret) {
3948 break;
3949 }
3950 }
3951 rcu_read_unlock();
3952 return ret;
3953}
3954
3955int qemu_ram_foreach_migratable_block(RAMBlockIterFunc func, void *opaque)
3956{
3957 RAMBlock *block;
3958 int ret = 0;
3959
3960 rcu_read_lock();
3961 RAMBLOCK_FOREACH(block) {
3962 if (!qemu_ram_is_migratable(block)) {
3963 continue;
3964 }
3965 ret = func(block->idstr, block->host, block->offset,
3966 block->used_length, opaque);
3967 if (ret) {
3968 break;
3969 }
3970 }
3971 rcu_read_unlock();
3972 return ret;
3973}
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3984{
3985 int ret = -1;
3986
3987 uint8_t *host_startaddr = rb->host + start;
3988
3989 if ((uintptr_t)host_startaddr & (rb->page_size - 1)) {
3990 error_report("ram_block_discard_range: Unaligned start address: %p",
3991 host_startaddr);
3992 goto err;
3993 }
3994
3995 if ((start + length) <= rb->used_length) {
3996 bool need_madvise, need_fallocate;
3997 uint8_t *host_endaddr = host_startaddr + length;
3998 if ((uintptr_t)host_endaddr & (rb->page_size - 1)) {
3999 error_report("ram_block_discard_range: Unaligned end address: %p",
4000 host_endaddr);
4001 goto err;
4002 }
4003
4004 errno = ENOTSUP;
4005
4006
4007
4008
4009
4010 need_madvise = (rb->page_size == qemu_host_page_size);
4011 need_fallocate = rb->fd != -1;
4012 if (need_fallocate) {
4013
4014
4015
4016
4017#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
4018 ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
4019 start, length);
4020 if (ret) {
4021 ret = -errno;
4022 error_report("ram_block_discard_range: Failed to fallocate "
4023 "%s:%" PRIx64 " +%zx (%d)",
4024 rb->idstr, start, length, ret);
4025 goto err;
4026 }
4027#else
4028 ret = -ENOSYS;
4029 error_report("ram_block_discard_range: fallocate not available/file"
4030 "%s:%" PRIx64 " +%zx (%d)",
4031 rb->idstr, start, length, ret);
4032 goto err;
4033#endif
4034 }
4035 if (need_madvise) {
4036
4037
4038
4039
4040
4041#if defined(CONFIG_MADVISE)
4042 ret = madvise(host_startaddr, length, MADV_DONTNEED);
4043 if (ret) {
4044 ret = -errno;
4045 error_report("ram_block_discard_range: Failed to discard range "
4046 "%s:%" PRIx64 " +%zx (%d)",
4047 rb->idstr, start, length, ret);
4048 goto err;
4049 }
4050#else
4051 ret = -ENOSYS;
4052 error_report("ram_block_discard_range: MADVISE not available"
4053 "%s:%" PRIx64 " +%zx (%d)",
4054 rb->idstr, start, length, ret);
4055 goto err;
4056#endif
4057 }
4058 trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
4059 need_madvise, need_fallocate, ret);
4060 } else {
4061 error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
4062 "/%zx/" RAM_ADDR_FMT")",
4063 rb->idstr, start, length, rb->used_length);
4064 }
4065
4066err:
4067 return ret;
4068}
4069
4070bool ramblock_is_pmem(RAMBlock *rb)
4071{
4072 return rb->flags & RAM_PMEM;
4073}
4074
4075#endif
4076
4077void page_size_init(void)
4078{
4079
4080
4081 if (qemu_host_page_size == 0) {
4082 qemu_host_page_size = qemu_real_host_page_size;
4083 }
4084 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
4085 qemu_host_page_size = TARGET_PAGE_SIZE;
4086 }
4087 qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
4088}
4089
4090#if !defined(CONFIG_USER_ONLY)
4091
4092static void mtree_print_phys_entries(fprintf_function mon, void *f,
4093 int start, int end, int skip, int ptr)
4094{
4095 if (start == end - 1) {
4096 mon(f, "\t%3d ", start);
4097 } else {
4098 mon(f, "\t%3d..%-3d ", start, end - 1);
4099 }
4100 mon(f, " skip=%d ", skip);
4101 if (ptr == PHYS_MAP_NODE_NIL) {
4102 mon(f, " ptr=NIL");
4103 } else if (!skip) {
4104 mon(f, " ptr=#%d", ptr);
4105 } else {
4106 mon(f, " ptr=[%d]", ptr);
4107 }
4108 mon(f, "\n");
4109}
4110
4111#define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
4112 int128_sub((size), int128_one())) : 0)
4113
4114void mtree_print_dispatch(fprintf_function mon, void *f,
4115 AddressSpaceDispatch *d, MemoryRegion *root)
4116{
4117 int i;
4118
4119 mon(f, " Dispatch\n");
4120 mon(f, " Physical sections\n");
4121
4122 for (i = 0; i < d->map.sections_nb; ++i) {
4123 MemoryRegionSection *s = d->map.sections + i;
4124 const char *names[] = { " [unassigned]", " [not dirty]",
4125 " [ROM]", " [watch]" };
4126
4127 mon(f, " #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx " %s%s%s%s%s",
4128 i,
4129 s->offset_within_address_space,
4130 s->offset_within_address_space + MR_SIZE(s->mr->size),
4131 s->mr->name ? s->mr->name : "(noname)",
4132 i < ARRAY_SIZE(names) ? names[i] : "",
4133 s->mr == root ? " [ROOT]" : "",
4134 s == d->mru_section ? " [MRU]" : "",
4135 s->mr->is_iommu ? " [iommu]" : "");
4136
4137 if (s->mr->alias) {
4138 mon(f, " alias=%s", s->mr->alias->name ?
4139 s->mr->alias->name : "noname");
4140 }
4141 mon(f, "\n");
4142 }
4143
4144 mon(f, " Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
4145 P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
4146 for (i = 0; i < d->map.nodes_nb; ++i) {
4147 int j, jprev;
4148 PhysPageEntry prev;
4149 Node *n = d->map.nodes + i;
4150
4151 mon(f, " [%d]\n", i);
4152
4153 for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
4154 PhysPageEntry *pe = *n + j;
4155
4156 if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
4157 continue;
4158 }
4159
4160 mtree_print_phys_entries(mon, f, jprev, j, prev.skip, prev.ptr);
4161
4162 jprev = j;
4163 prev = *pe;
4164 }
4165
4166 if (jprev != ARRAY_SIZE(*n)) {
4167 mtree_print_phys_entries(mon, f, jprev, j, prev.skip, prev.ptr);
4168 }
4169 }
4170}
4171
4172#endif
4173