1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include "qemu/osdep.h"
26#include "qemu/units.h"
27#include "qemu/madvise.h"
28#include "qemu/mprotect.h"
29#include "qemu/memalign.h"
30#include "qemu/cacheinfo.h"
31#include "qapi/error.h"
32#include "exec/exec-all.h"
33#include "tcg/tcg.h"
34#include "tcg-internal.h"
35
36
37struct tcg_region_tree {
38 QemuMutex lock;
39 GTree *tree;
40
41};
42
43
44
45
46
47
48
49struct tcg_region_state {
50 QemuMutex lock;
51
52
53 void *start_aligned;
54 void *after_prologue;
55 size_t n;
56 size_t size;
57 size_t stride;
58 size_t total_size;
59
60
61 size_t current;
62 size_t agg_size_full;
63};
64
65static struct tcg_region_state region;
66
67
68
69
70
71
72static void *region_trees;
73static size_t tree_size;
74
75bool in_code_gen_buffer(const void *p)
76{
77
78
79
80
81
82 return (size_t)(p - region.start_aligned) <= region.total_size;
83}
84
85#ifdef CONFIG_DEBUG_TCG
86const void *tcg_splitwx_to_rx(void *rw)
87{
88
89 if (rw) {
90 g_assert(in_code_gen_buffer(rw));
91 rw += tcg_splitwx_diff;
92 }
93 return rw;
94}
95
96void *tcg_splitwx_to_rw(const void *rx)
97{
98
99 if (rx) {
100 rx -= tcg_splitwx_diff;
101
102 g_assert(in_code_gen_buffer(rx));
103 }
104 return (void *)rx;
105}
106#endif
107
108
109static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s)
110{
111 if (ptr >= s->ptr + s->size) {
112 return 1;
113 } else if (ptr < s->ptr) {
114 return -1;
115 }
116 return 0;
117}
118
119static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp, gpointer userdata)
120{
121 const struct tb_tc *a = ap;
122 const struct tb_tc *b = bp;
123
124
125
126
127
128
129 if (likely(a->size && b->size)) {
130 if (a->ptr > b->ptr) {
131 return 1;
132 } else if (a->ptr < b->ptr) {
133 return -1;
134 }
135
136 g_assert(a->size == b->size);
137 return 0;
138 }
139
140
141
142
143
144 if (likely(a->size == 0)) {
145 return ptr_cmp_tb_tc(a->ptr, b);
146 }
147 return ptr_cmp_tb_tc(b->ptr, a);
148}
149
150static void tb_destroy(gpointer value)
151{
152 TranslationBlock *tb = value;
153 qemu_spin_destroy(&tb->jmp_lock);
154}
155
156static void tcg_region_trees_init(void)
157{
158 size_t i;
159
160 tree_size = ROUND_UP(sizeof(struct tcg_region_tree), qemu_dcache_linesize);
161 region_trees = qemu_memalign(qemu_dcache_linesize, region.n * tree_size);
162 for (i = 0; i < region.n; i++) {
163 struct tcg_region_tree *rt = region_trees + i * tree_size;
164
165 qemu_mutex_init(&rt->lock);
166 rt->tree = g_tree_new_full(tb_tc_cmp, NULL, NULL, tb_destroy);
167 }
168}
169
170static struct tcg_region_tree *tc_ptr_to_region_tree(const void *p)
171{
172 size_t region_idx;
173
174
175
176
177
178 if (!in_code_gen_buffer(p)) {
179 p -= tcg_splitwx_diff;
180 if (!in_code_gen_buffer(p)) {
181 return NULL;
182 }
183 }
184
185 if (p < region.start_aligned) {
186 region_idx = 0;
187 } else {
188 ptrdiff_t offset = p - region.start_aligned;
189
190 if (offset > region.stride * (region.n - 1)) {
191 region_idx = region.n - 1;
192 } else {
193 region_idx = offset / region.stride;
194 }
195 }
196 return region_trees + region_idx * tree_size;
197}
198
199void tcg_tb_insert(TranslationBlock *tb)
200{
201 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
202
203 g_assert(rt != NULL);
204 qemu_mutex_lock(&rt->lock);
205 g_tree_insert(rt->tree, &tb->tc, tb);
206 qemu_mutex_unlock(&rt->lock);
207}
208
209void tcg_tb_remove(TranslationBlock *tb)
210{
211 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
212
213 g_assert(rt != NULL);
214 qemu_mutex_lock(&rt->lock);
215 g_tree_remove(rt->tree, &tb->tc);
216 qemu_mutex_unlock(&rt->lock);
217}
218
219
220
221
222
223
224TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr)
225{
226 struct tcg_region_tree *rt = tc_ptr_to_region_tree((void *)tc_ptr);
227 TranslationBlock *tb;
228 struct tb_tc s = { .ptr = (void *)tc_ptr };
229
230 if (rt == NULL) {
231 return NULL;
232 }
233
234 qemu_mutex_lock(&rt->lock);
235 tb = g_tree_lookup(rt->tree, &s);
236 qemu_mutex_unlock(&rt->lock);
237 return tb;
238}
239
240static void tcg_region_tree_lock_all(void)
241{
242 size_t i;
243
244 for (i = 0; i < region.n; i++) {
245 struct tcg_region_tree *rt = region_trees + i * tree_size;
246
247 qemu_mutex_lock(&rt->lock);
248 }
249}
250
251static void tcg_region_tree_unlock_all(void)
252{
253 size_t i;
254
255 for (i = 0; i < region.n; i++) {
256 struct tcg_region_tree *rt = region_trees + i * tree_size;
257
258 qemu_mutex_unlock(&rt->lock);
259 }
260}
261
262void tcg_tb_foreach(GTraverseFunc func, gpointer user_data)
263{
264 size_t i;
265
266 tcg_region_tree_lock_all();
267 for (i = 0; i < region.n; i++) {
268 struct tcg_region_tree *rt = region_trees + i * tree_size;
269
270 g_tree_foreach(rt->tree, func, user_data);
271 }
272 tcg_region_tree_unlock_all();
273}
274
275size_t tcg_nb_tbs(void)
276{
277 size_t nb_tbs = 0;
278 size_t i;
279
280 tcg_region_tree_lock_all();
281 for (i = 0; i < region.n; i++) {
282 struct tcg_region_tree *rt = region_trees + i * tree_size;
283
284 nb_tbs += g_tree_nnodes(rt->tree);
285 }
286 tcg_region_tree_unlock_all();
287 return nb_tbs;
288}
289
290static void tcg_region_tree_reset_all(void)
291{
292 size_t i;
293
294 tcg_region_tree_lock_all();
295 for (i = 0; i < region.n; i++) {
296 struct tcg_region_tree *rt = region_trees + i * tree_size;
297
298
299 g_tree_ref(rt->tree);
300 g_tree_destroy(rt->tree);
301 }
302 tcg_region_tree_unlock_all();
303}
304
305static void tcg_region_bounds(size_t curr_region, void **pstart, void **pend)
306{
307 void *start, *end;
308
309 start = region.start_aligned + curr_region * region.stride;
310 end = start + region.size;
311
312 if (curr_region == 0) {
313 start = region.after_prologue;
314 }
315
316 if (curr_region == region.n - 1) {
317 end = region.start_aligned + region.total_size;
318 }
319
320 *pstart = start;
321 *pend = end;
322}
323
324static void tcg_region_assign(TCGContext *s, size_t curr_region)
325{
326 void *start, *end;
327
328 tcg_region_bounds(curr_region, &start, &end);
329
330 s->code_gen_buffer = start;
331 s->code_gen_ptr = start;
332 s->code_gen_buffer_size = end - start;
333 s->code_gen_highwater = end - TCG_HIGHWATER;
334}
335
336static bool tcg_region_alloc__locked(TCGContext *s)
337{
338 if (region.current == region.n) {
339 return true;
340 }
341 tcg_region_assign(s, region.current);
342 region.current++;
343 return false;
344}
345
346
347
348
349
350bool tcg_region_alloc(TCGContext *s)
351{
352 bool err;
353
354 size_t size_full = s->code_gen_buffer_size;
355
356 qemu_mutex_lock(®ion.lock);
357 err = tcg_region_alloc__locked(s);
358 if (!err) {
359 region.agg_size_full += size_full - TCG_HIGHWATER;
360 }
361 qemu_mutex_unlock(®ion.lock);
362 return err;
363}
364
365
366
367
368
369static void tcg_region_initial_alloc__locked(TCGContext *s)
370{
371 bool err = tcg_region_alloc__locked(s);
372 g_assert(!err);
373}
374
375void tcg_region_initial_alloc(TCGContext *s)
376{
377 qemu_mutex_lock(®ion.lock);
378 tcg_region_initial_alloc__locked(s);
379 qemu_mutex_unlock(®ion.lock);
380}
381
382
383void tcg_region_reset_all(void)
384{
385 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
386 unsigned int i;
387
388 qemu_mutex_lock(®ion.lock);
389 region.current = 0;
390 region.agg_size_full = 0;
391
392 for (i = 0; i < n_ctxs; i++) {
393 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
394 tcg_region_initial_alloc__locked(s);
395 }
396 qemu_mutex_unlock(®ion.lock);
397
398 tcg_region_tree_reset_all();
399}
400
401static size_t tcg_n_regions(size_t tb_size, unsigned max_cpus)
402{
403#ifdef CONFIG_USER_ONLY
404 return 1;
405#else
406 size_t n_regions;
407
408
409
410
411
412
413
414
415 if (max_cpus == 1 || !qemu_tcg_mttcg_enabled()) {
416 return 1;
417 }
418
419
420
421
422
423 n_regions = tb_size / (2 * MiB);
424 if (n_regions <= max_cpus) {
425 return max_cpus;
426 }
427 return MIN(n_regions, max_cpus * 8);
428#endif
429}
430
431
432
433
434
435
436
437
438
439
440#define MIN_CODE_GEN_BUFFER_SIZE (1 * MiB)
441
442#if TCG_TARGET_REG_BITS == 32
443#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32 * MiB)
444#ifdef CONFIG_USER_ONLY
445
446
447
448
449
450#define USE_STATIC_CODE_GEN_BUFFER
451#endif
452#else
453#ifdef CONFIG_USER_ONLY
454
455
456
457
458
459#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (128 * MiB)
460#else
461
462
463
464
465
466#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (1 * GiB)
467#endif
468#endif
469
470#define DEFAULT_CODE_GEN_BUFFER_SIZE \
471 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
472 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
473
474#ifdef USE_STATIC_CODE_GEN_BUFFER
475static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
476 __attribute__((aligned(CODE_GEN_ALIGN)));
477
478static int alloc_code_gen_buffer(size_t tb_size, int splitwx, Error **errp)
479{
480 void *buf, *end;
481 size_t size;
482
483 if (splitwx > 0) {
484 error_setg(errp, "jit split-wx not supported");
485 return -1;
486 }
487
488
489 buf = static_code_gen_buffer;
490 end = static_code_gen_buffer + sizeof(static_code_gen_buffer);
491 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size);
492 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size);
493
494 size = end - buf;
495
496
497 if (size > tb_size) {
498 size = QEMU_ALIGN_DOWN(tb_size, qemu_real_host_page_size);
499 }
500
501 region.start_aligned = buf;
502 region.total_size = size;
503
504 return PROT_READ | PROT_WRITE;
505}
506#elif defined(_WIN32)
507static int alloc_code_gen_buffer(size_t size, int splitwx, Error **errp)
508{
509 void *buf;
510
511 if (splitwx > 0) {
512 error_setg(errp, "jit split-wx not supported");
513 return -1;
514 }
515
516 buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT,
517 PAGE_EXECUTE_READWRITE);
518 if (buf == NULL) {
519 error_setg_win32(errp, GetLastError(),
520 "allocate %zu bytes for jit buffer", size);
521 return false;
522 }
523
524 region.start_aligned = buf;
525 region.total_size = size;
526
527 return PAGE_READ | PAGE_WRITE | PAGE_EXEC;
528}
529#else
530static int alloc_code_gen_buffer_anon(size_t size, int prot,
531 int flags, Error **errp)
532{
533 void *buf;
534
535 buf = mmap(NULL, size, prot, flags, -1, 0);
536 if (buf == MAP_FAILED) {
537 error_setg_errno(errp, errno,
538 "allocate %zu bytes for jit buffer", size);
539 return -1;
540 }
541
542 region.start_aligned = buf;
543 region.total_size = size;
544 return prot;
545}
546
547#ifndef CONFIG_TCG_INTERPRETER
548#ifdef CONFIG_POSIX
549#include "qemu/memfd.h"
550
551static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp)
552{
553 void *buf_rw = NULL, *buf_rx = MAP_FAILED;
554 int fd = -1;
555
556 buf_rw = qemu_memfd_alloc("tcg-jit", size, 0, &fd, errp);
557 if (buf_rw == NULL) {
558 goto fail;
559 }
560
561 buf_rx = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0);
562 if (buf_rx == MAP_FAILED) {
563 goto fail_rx;
564 }
565
566 close(fd);
567 region.start_aligned = buf_rw;
568 region.total_size = size;
569 tcg_splitwx_diff = buf_rx - buf_rw;
570
571 return PROT_READ | PROT_WRITE;
572
573 fail_rx:
574 error_setg_errno(errp, errno, "failed to map shared memory for execute");
575 fail:
576 if (buf_rx != MAP_FAILED) {
577 munmap(buf_rx, size);
578 }
579 if (buf_rw) {
580 munmap(buf_rw, size);
581 }
582 if (fd >= 0) {
583 close(fd);
584 }
585 return -1;
586}
587#endif
588
589#ifdef CONFIG_DARWIN
590#include <mach/mach.h>
591
592extern kern_return_t mach_vm_remap(vm_map_t target_task,
593 mach_vm_address_t *target_address,
594 mach_vm_size_t size,
595 mach_vm_offset_t mask,
596 int flags,
597 vm_map_t src_task,
598 mach_vm_address_t src_address,
599 boolean_t copy,
600 vm_prot_t *cur_protection,
601 vm_prot_t *max_protection,
602 vm_inherit_t inheritance);
603
604static int alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp)
605{
606 kern_return_t ret;
607 mach_vm_address_t buf_rw, buf_rx;
608 vm_prot_t cur_prot, max_prot;
609
610
611 if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE,
612 MAP_PRIVATE | MAP_ANONYMOUS, errp)) {
613 return -1;
614 }
615
616 buf_rw = (mach_vm_address_t)region.start_aligned;
617 buf_rx = 0;
618 ret = mach_vm_remap(mach_task_self(),
619 &buf_rx,
620 size,
621 0,
622 VM_FLAGS_ANYWHERE,
623 mach_task_self(),
624 buf_rw,
625 false,
626 &cur_prot,
627 &max_prot,
628 VM_INHERIT_NONE);
629 if (ret != KERN_SUCCESS) {
630
631 error_setg(errp, "vm_remap for jit splitwx failed");
632 munmap((void *)buf_rw, size);
633 return -1;
634 }
635
636 if (mprotect((void *)buf_rx, size, PROT_READ | PROT_EXEC) != 0) {
637 error_setg_errno(errp, errno, "mprotect for jit splitwx");
638 munmap((void *)buf_rx, size);
639 munmap((void *)buf_rw, size);
640 return -1;
641 }
642
643 tcg_splitwx_diff = buf_rx - buf_rw;
644 return PROT_READ | PROT_WRITE;
645}
646#endif
647#endif
648
649static int alloc_code_gen_buffer_splitwx(size_t size, Error **errp)
650{
651#ifndef CONFIG_TCG_INTERPRETER
652# ifdef CONFIG_DARWIN
653 return alloc_code_gen_buffer_splitwx_vmremap(size, errp);
654# endif
655# ifdef CONFIG_POSIX
656 return alloc_code_gen_buffer_splitwx_memfd(size, errp);
657# endif
658#endif
659 error_setg(errp, "jit split-wx not supported");
660 return -1;
661}
662
663static int alloc_code_gen_buffer(size_t size, int splitwx, Error **errp)
664{
665 ERRP_GUARD();
666 int prot, flags;
667
668 if (splitwx) {
669 prot = alloc_code_gen_buffer_splitwx(size, errp);
670 if (prot >= 0) {
671 return prot;
672 }
673
674
675
676
677 if (splitwx > 0) {
678 return -1;
679 }
680 error_free_or_abort(errp);
681 }
682
683
684
685
686
687
688
689 prot = PROT_NONE;
690 flags = MAP_PRIVATE | MAP_ANONYMOUS;
691#ifdef CONFIG_DARWIN
692
693 if (!splitwx) {
694 flags |= MAP_JIT;
695 }
696#endif
697
698 return alloc_code_gen_buffer_anon(size, prot, flags, errp);
699}
700#endif
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730void tcg_region_init(size_t tb_size, int splitwx, unsigned max_cpus)
731{
732 const size_t page_size = qemu_real_host_page_size;
733 size_t region_size;
734 int have_prot, need_prot;
735
736
737 if (tb_size == 0) {
738 size_t phys_mem = qemu_get_host_physmem();
739 if (phys_mem == 0) {
740 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
741 } else {
742 tb_size = QEMU_ALIGN_DOWN(phys_mem / 8, page_size);
743 tb_size = MIN(DEFAULT_CODE_GEN_BUFFER_SIZE, tb_size);
744 }
745 }
746 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
747 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
748 }
749 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
750 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
751 }
752
753 have_prot = alloc_code_gen_buffer(tb_size, splitwx, &error_fatal);
754 assert(have_prot >= 0);
755
756
757 qemu_madvise(region.start_aligned, region.total_size, QEMU_MADV_HUGEPAGE);
758 if (tcg_splitwx_diff) {
759 qemu_madvise(region.start_aligned + tcg_splitwx_diff,
760 region.total_size, QEMU_MADV_HUGEPAGE);
761 }
762
763
764
765
766
767
768 region.n = tcg_n_regions(tb_size, max_cpus);
769 region_size = tb_size / region.n;
770 region_size = QEMU_ALIGN_DOWN(region_size, page_size);
771
772
773 g_assert(region_size >= 2 * page_size);
774 region.stride = region_size;
775
776
777 region.size = region_size - page_size;
778 region.total_size -= page_size;
779
780
781
782
783
784
785 region.after_prologue = region.start_aligned;
786
787
788 qemu_mutex_init(®ion.lock);
789
790
791
792
793
794
795
796 need_prot = PAGE_READ | PAGE_WRITE;
797#ifndef CONFIG_TCG_INTERPRETER
798 if (tcg_splitwx_diff == 0) {
799 need_prot |= PAGE_EXEC;
800 }
801#endif
802 for (size_t i = 0, n = region.n; i < n; i++) {
803 void *start, *end;
804
805 tcg_region_bounds(i, &start, &end);
806 if (have_prot != need_prot) {
807 int rc;
808
809 if (need_prot == (PAGE_READ | PAGE_WRITE | PAGE_EXEC)) {
810 rc = qemu_mprotect_rwx(start, end - start);
811 } else if (need_prot == (PAGE_READ | PAGE_WRITE)) {
812 rc = qemu_mprotect_rw(start, end - start);
813 } else {
814 g_assert_not_reached();
815 }
816 if (rc) {
817 error_setg_errno(&error_fatal, errno,
818 "mprotect of jit buffer");
819 }
820 }
821 if (have_prot != 0) {
822
823 (void)qemu_mprotect_none(end, page_size);
824 }
825 }
826
827 tcg_region_trees_init();
828
829
830
831
832
833
834 tcg_region_initial_alloc__locked(&tcg_init_ctx);
835}
836
837void tcg_region_prologue_set(TCGContext *s)
838{
839
840 g_assert(region.start_aligned == s->code_gen_buffer);
841 region.after_prologue = s->code_ptr;
842
843
844 tcg_region_assign(s, 0);
845
846
847 tcg_register_jit(tcg_splitwx_to_rx(region.after_prologue),
848 region.start_aligned + region.total_size -
849 region.after_prologue);
850}
851
852
853
854
855
856
857
858
859size_t tcg_code_size(void)
860{
861 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
862 unsigned int i;
863 size_t total;
864
865 qemu_mutex_lock(®ion.lock);
866 total = region.agg_size_full;
867 for (i = 0; i < n_ctxs; i++) {
868 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
869 size_t size;
870
871 size = qatomic_read(&s->code_gen_ptr) - s->code_gen_buffer;
872 g_assert(size <= s->code_gen_buffer_size);
873 total += size;
874 }
875 qemu_mutex_unlock(®ion.lock);
876 return total;
877}
878
879
880
881
882
883
884size_t tcg_code_capacity(void)
885{
886 size_t guard_size, capacity;
887
888
889 guard_size = region.stride - region.size;
890 capacity = region.total_size;
891 capacity -= (region.n - 1) * guard_size;
892 capacity -= region.n * TCG_HIGHWATER;
893
894 return capacity;
895}
896