1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef TCG_H
26#define TCG_H
27
28#include "cpu.h"
29#include "exec/memop.h"
30#include "exec/tb-context.h"
31#include "qemu/bitops.h"
32#include "qemu/plugin.h"
33#include "qemu/queue.h"
34#include "tcg/tcg-mo.h"
35#include "tcg-target.h"
36#include "qemu/int128.h"
37
38
39#define MAX_OP_PER_INSTR 266
40
41#if HOST_LONG_BITS == 32
42#define MAX_OPC_PARAM_PER_ARG 2
43#else
44#define MAX_OPC_PARAM_PER_ARG 1
45#endif
46#define MAX_OPC_PARAM_IARGS 6
47#define MAX_OPC_PARAM_OARGS 1
48#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
49
50
51
52
53#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
54
55#define CPU_TEMP_BUF_NLONGS 128
56
57
58#ifndef TCG_TARGET_REG_BITS
59# if UINTPTR_MAX == UINT32_MAX
60# define TCG_TARGET_REG_BITS 32
61# elif UINTPTR_MAX == UINT64_MAX
62# define TCG_TARGET_REG_BITS 64
63# else
64# error Unknown pointer size for tcg target
65# endif
66#endif
67
68#if TCG_TARGET_REG_BITS == 32
69typedef int32_t tcg_target_long;
70typedef uint32_t tcg_target_ulong;
71#define TCG_PRIlx PRIx32
72#define TCG_PRIld PRId32
73#elif TCG_TARGET_REG_BITS == 64
74typedef int64_t tcg_target_long;
75typedef uint64_t tcg_target_ulong;
76#define TCG_PRIlx PRIx64
77#define TCG_PRIld PRId64
78#else
79#error unsupported
80#endif
81
82
83
84
85#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
86#define TCG_OVERSIZED_GUEST 1
87#else
88#define TCG_OVERSIZED_GUEST 0
89#endif
90
91#if TCG_TARGET_NB_REGS <= 32
92typedef uint32_t TCGRegSet;
93#elif TCG_TARGET_NB_REGS <= 64
94typedef uint64_t TCGRegSet;
95#else
96#error unsupported
97#endif
98
99#if TCG_TARGET_REG_BITS == 32
100
101#define TCG_TARGET_HAS_extrl_i64_i32 0
102#define TCG_TARGET_HAS_extrh_i64_i32 0
103#define TCG_TARGET_HAS_div_i64 0
104#define TCG_TARGET_HAS_rem_i64 0
105#define TCG_TARGET_HAS_div2_i64 0
106#define TCG_TARGET_HAS_rot_i64 0
107#define TCG_TARGET_HAS_ext8s_i64 0
108#define TCG_TARGET_HAS_ext16s_i64 0
109#define TCG_TARGET_HAS_ext32s_i64 0
110#define TCG_TARGET_HAS_ext8u_i64 0
111#define TCG_TARGET_HAS_ext16u_i64 0
112#define TCG_TARGET_HAS_ext32u_i64 0
113#define TCG_TARGET_HAS_bswap16_i64 0
114#define TCG_TARGET_HAS_bswap32_i64 0
115#define TCG_TARGET_HAS_bswap64_i64 0
116#define TCG_TARGET_HAS_neg_i64 0
117#define TCG_TARGET_HAS_not_i64 0
118#define TCG_TARGET_HAS_andc_i64 0
119#define TCG_TARGET_HAS_orc_i64 0
120#define TCG_TARGET_HAS_eqv_i64 0
121#define TCG_TARGET_HAS_nand_i64 0
122#define TCG_TARGET_HAS_nor_i64 0
123#define TCG_TARGET_HAS_clz_i64 0
124#define TCG_TARGET_HAS_ctz_i64 0
125#define TCG_TARGET_HAS_ctpop_i64 0
126#define TCG_TARGET_HAS_deposit_i64 0
127#define TCG_TARGET_HAS_extract_i64 0
128#define TCG_TARGET_HAS_sextract_i64 0
129#define TCG_TARGET_HAS_extract2_i64 0
130#define TCG_TARGET_HAS_movcond_i64 0
131#define TCG_TARGET_HAS_add2_i64 0
132#define TCG_TARGET_HAS_sub2_i64 0
133#define TCG_TARGET_HAS_mulu2_i64 0
134#define TCG_TARGET_HAS_muls2_i64 0
135#define TCG_TARGET_HAS_muluh_i64 0
136#define TCG_TARGET_HAS_mulsh_i64 0
137
138#define TCG_TARGET_HAS_add2_i32 1
139#define TCG_TARGET_HAS_sub2_i32 1
140#endif
141
142#ifndef TCG_TARGET_deposit_i32_valid
143#define TCG_TARGET_deposit_i32_valid(ofs, len) 1
144#endif
145#ifndef TCG_TARGET_deposit_i64_valid
146#define TCG_TARGET_deposit_i64_valid(ofs, len) 1
147#endif
148#ifndef TCG_TARGET_extract_i32_valid
149#define TCG_TARGET_extract_i32_valid(ofs, len) 1
150#endif
151#ifndef TCG_TARGET_extract_i64_valid
152#define TCG_TARGET_extract_i64_valid(ofs, len) 1
153#endif
154
155
156#if defined(TCG_TARGET_HAS_div_i32)
157#define TCG_TARGET_HAS_div2_i32 0
158#elif defined(TCG_TARGET_HAS_div2_i32)
159#define TCG_TARGET_HAS_div_i32 0
160#define TCG_TARGET_HAS_rem_i32 0
161#endif
162#if defined(TCG_TARGET_HAS_div_i64)
163#define TCG_TARGET_HAS_div2_i64 0
164#elif defined(TCG_TARGET_HAS_div2_i64)
165#define TCG_TARGET_HAS_div_i64 0
166#define TCG_TARGET_HAS_rem_i64 0
167#endif
168
169
170#if TCG_TARGET_REG_BITS == 32 \
171 && !(defined(TCG_TARGET_HAS_mulu2_i32) \
172 || defined(TCG_TARGET_HAS_muluh_i32))
173# error "Missing unsigned widening multiply"
174#endif
175
176#if !defined(TCG_TARGET_HAS_v64) \
177 && !defined(TCG_TARGET_HAS_v128) \
178 && !defined(TCG_TARGET_HAS_v256)
179#define TCG_TARGET_MAYBE_vec 0
180#define TCG_TARGET_HAS_abs_vec 0
181#define TCG_TARGET_HAS_neg_vec 0
182#define TCG_TARGET_HAS_not_vec 0
183#define TCG_TARGET_HAS_andc_vec 0
184#define TCG_TARGET_HAS_orc_vec 0
185#define TCG_TARGET_HAS_roti_vec 0
186#define TCG_TARGET_HAS_rots_vec 0
187#define TCG_TARGET_HAS_rotv_vec 0
188#define TCG_TARGET_HAS_shi_vec 0
189#define TCG_TARGET_HAS_shs_vec 0
190#define TCG_TARGET_HAS_shv_vec 0
191#define TCG_TARGET_HAS_mul_vec 0
192#define TCG_TARGET_HAS_sat_vec 0
193#define TCG_TARGET_HAS_minmax_vec 0
194#define TCG_TARGET_HAS_bitsel_vec 0
195#define TCG_TARGET_HAS_cmpsel_vec 0
196#else
197#define TCG_TARGET_MAYBE_vec 1
198#endif
199#ifndef TCG_TARGET_HAS_v64
200#define TCG_TARGET_HAS_v64 0
201#endif
202#ifndef TCG_TARGET_HAS_v128
203#define TCG_TARGET_HAS_v128 0
204#endif
205#ifndef TCG_TARGET_HAS_v256
206#define TCG_TARGET_HAS_v256 0
207#endif
208
209#ifndef TARGET_INSN_START_EXTRA_WORDS
210# define TARGET_INSN_START_WORDS 1
211#else
212# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
213#endif
214
215typedef enum TCGOpcode {
216#define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
217#include "tcg/tcg-opc.h"
218#undef DEF
219 NB_OPS,
220} TCGOpcode;
221
222#define tcg_regset_set_reg(d, r) ((d) |= (TCGRegSet)1 << (r))
223#define tcg_regset_reset_reg(d, r) ((d) &= ~((TCGRegSet)1 << (r)))
224#define tcg_regset_test_reg(d, r) (((d) >> (r)) & 1)
225
226#ifndef TCG_TARGET_INSN_UNIT_SIZE
227# error "Missing TCG_TARGET_INSN_UNIT_SIZE"
228#elif TCG_TARGET_INSN_UNIT_SIZE == 1
229typedef uint8_t tcg_insn_unit;
230#elif TCG_TARGET_INSN_UNIT_SIZE == 2
231typedef uint16_t tcg_insn_unit;
232#elif TCG_TARGET_INSN_UNIT_SIZE == 4
233typedef uint32_t tcg_insn_unit;
234#elif TCG_TARGET_INSN_UNIT_SIZE == 8
235typedef uint64_t tcg_insn_unit;
236#else
237
238#endif
239
240
241#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
242# define tcg_debug_assert(X) do { assert(X); } while (0)
243#else
244# define tcg_debug_assert(X) \
245 do { if (!(X)) { __builtin_unreachable(); } } while (0)
246#endif
247
248typedef struct TCGRelocation TCGRelocation;
249struct TCGRelocation {
250 QSIMPLEQ_ENTRY(TCGRelocation) next;
251 tcg_insn_unit *ptr;
252 intptr_t addend;
253 int type;
254};
255
256typedef struct TCGLabel TCGLabel;
257struct TCGLabel {
258 unsigned present : 1;
259 unsigned has_value : 1;
260 unsigned id : 14;
261 unsigned refs : 16;
262 union {
263 uintptr_t value;
264 tcg_insn_unit *value_ptr;
265 } u;
266 QSIMPLEQ_HEAD(, TCGRelocation) relocs;
267 QSIMPLEQ_ENTRY(TCGLabel) next;
268};
269
270typedef struct TCGPool {
271 struct TCGPool *next;
272 int size;
273 uint8_t data[] __attribute__ ((aligned));
274} TCGPool;
275
276#define TCG_POOL_CHUNK_SIZE 32768
277
278#define TCG_MAX_TEMPS 512
279#define TCG_MAX_INSNS 512
280
281
282
283#define TCG_STATIC_CALL_ARGS_SIZE 128
284
285typedef enum TCGType {
286 TCG_TYPE_I32,
287 TCG_TYPE_I64,
288
289 TCG_TYPE_V64,
290 TCG_TYPE_V128,
291 TCG_TYPE_V256,
292
293 TCG_TYPE_COUNT,
294
295
296#if TCG_TARGET_REG_BITS == 32
297 TCG_TYPE_REG = TCG_TYPE_I32,
298#else
299 TCG_TYPE_REG = TCG_TYPE_I64,
300#endif
301
302
303#if UINTPTR_MAX == UINT32_MAX
304 TCG_TYPE_PTR = TCG_TYPE_I32,
305#else
306 TCG_TYPE_PTR = TCG_TYPE_I64,
307#endif
308
309
310#if TARGET_LONG_BITS == 64
311 TCG_TYPE_TL = TCG_TYPE_I64,
312#else
313 TCG_TYPE_TL = TCG_TYPE_I32,
314#endif
315} TCGType;
316
317
318
319
320
321
322
323static inline unsigned get_alignment_bits(MemOp memop)
324{
325 unsigned a = memop & MO_AMASK;
326
327 if (a == MO_UNALN) {
328
329 a = 0;
330 } else if (a == MO_ALIGN) {
331
332 a = memop & MO_SIZE;
333 } else {
334
335 a = a >> MO_ASHIFT;
336 }
337#if defined(CONFIG_SOFTMMU)
338
339 tcg_debug_assert((TLB_FLAGS_MASK & ((1 << a) - 1)) == 0);
340#endif
341 return a;
342}
343
344typedef tcg_target_ulong TCGArg;
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376typedef struct TCGv_i32_d *TCGv_i32;
377typedef struct TCGv_i64_d *TCGv_i64;
378typedef struct TCGv_ptr_d *TCGv_ptr;
379typedef struct TCGv_vec_d *TCGv_vec;
380typedef TCGv_ptr TCGv_env;
381#if TARGET_LONG_BITS == 32
382#define TCGv TCGv_i32
383#elif TARGET_LONG_BITS == 64
384#define TCGv TCGv_i64
385#else
386#error Unhandled TARGET_LONG_BITS value
387#endif
388
389
390
391
392#define TCG_CALL_NO_READ_GLOBALS 0x0001
393
394#define TCG_CALL_NO_WRITE_GLOBALS 0x0002
395
396#define TCG_CALL_NO_SIDE_EFFECTS 0x0004
397
398#define TCG_CALL_NO_RETURN 0x0008
399
400
401#define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS
402#define TCG_CALL_NO_WG TCG_CALL_NO_WRITE_GLOBALS
403#define TCG_CALL_NO_SE TCG_CALL_NO_SIDE_EFFECTS
404#define TCG_CALL_NO_RWG_SE (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
405#define TCG_CALL_NO_WG_SE (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
406
407
408#define TCG_CALL_DUMMY_ARG ((TCGArg)0)
409
410
411
412
413
414
415
416typedef enum {
417
418 TCG_COND_NEVER = 0 | 0 | 0 | 0,
419 TCG_COND_ALWAYS = 0 | 0 | 0 | 1,
420 TCG_COND_EQ = 8 | 0 | 0 | 0,
421 TCG_COND_NE = 8 | 0 | 0 | 1,
422
423 TCG_COND_LT = 0 | 0 | 2 | 0,
424 TCG_COND_GE = 0 | 0 | 2 | 1,
425 TCG_COND_LE = 8 | 0 | 2 | 0,
426 TCG_COND_GT = 8 | 0 | 2 | 1,
427
428 TCG_COND_LTU = 0 | 4 | 0 | 0,
429 TCG_COND_GEU = 0 | 4 | 0 | 1,
430 TCG_COND_LEU = 8 | 4 | 0 | 0,
431 TCG_COND_GTU = 8 | 4 | 0 | 1,
432} TCGCond;
433
434
435static inline TCGCond tcg_invert_cond(TCGCond c)
436{
437 return (TCGCond)(c ^ 1);
438}
439
440
441static inline TCGCond tcg_swap_cond(TCGCond c)
442{
443 return c & 6 ? (TCGCond)(c ^ 9) : c;
444}
445
446
447static inline TCGCond tcg_unsigned_cond(TCGCond c)
448{
449 return c & 2 ? (TCGCond)(c ^ 6) : c;
450}
451
452
453static inline TCGCond tcg_signed_cond(TCGCond c)
454{
455 return c & 4 ? (TCGCond)(c ^ 6) : c;
456}
457
458
459static inline bool is_unsigned_cond(TCGCond c)
460{
461 return (c & 4) != 0;
462}
463
464
465
466static inline TCGCond tcg_high_cond(TCGCond c)
467{
468 switch (c) {
469 case TCG_COND_GE:
470 case TCG_COND_LE:
471 case TCG_COND_GEU:
472 case TCG_COND_LEU:
473 return (TCGCond)(c ^ 8);
474 default:
475 return c;
476 }
477}
478
479typedef enum TCGTempVal {
480 TEMP_VAL_DEAD,
481 TEMP_VAL_REG,
482 TEMP_VAL_MEM,
483 TEMP_VAL_CONST,
484} TCGTempVal;
485
486typedef struct TCGTemp {
487 TCGReg reg:8;
488 TCGTempVal val_type:8;
489 TCGType base_type:8;
490 TCGType type:8;
491 unsigned int fixed_reg:1;
492 unsigned int indirect_reg:1;
493 unsigned int indirect_base:1;
494 unsigned int mem_coherent:1;
495 unsigned int mem_allocated:1;
496
497
498 unsigned int temp_global:1;
499
500
501
502 unsigned int temp_local:1;
503 unsigned int temp_allocated:1;
504
505 tcg_target_long val;
506 struct TCGTemp *mem_base;
507 intptr_t mem_offset;
508 const char *name;
509
510
511
512
513 uintptr_t state;
514 void *state_ptr;
515} TCGTemp;
516
517typedef struct TCGContext TCGContext;
518
519typedef struct TCGTempSet {
520 unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
521} TCGTempSet;
522
523
524
525
526
527#define DEAD_ARG 4
528#define SYNC_ARG 1
529typedef uint16_t TCGLifeData;
530
531
532
533typedef struct TCGOp {
534 TCGOpcode opc : 8;
535
536
537 unsigned param1 : 4;
538 unsigned param2 : 4;
539
540
541 unsigned life : 16;
542
543
544 QTAILQ_ENTRY(TCGOp) link;
545#ifdef CONFIG_PLUGIN
546 QSIMPLEQ_ENTRY(TCGOp) plugin_link;
547#endif
548
549
550 TCGArg args[MAX_OPC_PARAM];
551
552
553 TCGRegSet output_pref[2];
554} TCGOp;
555
556#define TCGOP_CALLI(X) (X)->param1
557#define TCGOP_CALLO(X) (X)->param2
558
559#define TCGOP_VECL(X) (X)->param1
560#define TCGOP_VECE(X) (X)->param2
561
562
563QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
564
565typedef struct TCGProfile {
566 int64_t cpu_exec_time;
567 int64_t tb_count1;
568 int64_t tb_count;
569 int64_t op_count;
570 int op_count_max;
571 int temp_count_max;
572 int64_t temp_count;
573 int64_t del_op_count;
574 int64_t code_in_len;
575 int64_t code_out_len;
576 int64_t search_out_len;
577 int64_t interm_time;
578 int64_t code_time;
579 int64_t la_time;
580 int64_t opt_time;
581 int64_t restore_count;
582 int64_t restore_time;
583 int64_t table_op_count[NB_OPS];
584} TCGProfile;
585
586struct TCGContext {
587 uint8_t *pool_cur, *pool_end;
588 TCGPool *pool_first, *pool_current, *pool_first_large;
589 int nb_labels;
590 int nb_globals;
591 int nb_temps;
592 int nb_indirects;
593 int nb_ops;
594
595
596 tcg_insn_unit *code_buf;
597 uint16_t *tb_jmp_reset_offset;
598 uintptr_t *tb_jmp_insn_offset;
599 uintptr_t *tb_jmp_target_addr;
600
601 TCGRegSet reserved_regs;
602 uint32_t tb_cflags;
603 intptr_t current_frame_offset;
604 intptr_t frame_start;
605 intptr_t frame_end;
606 TCGTemp *frame_temp;
607
608 tcg_insn_unit *code_ptr;
609
610#ifdef CONFIG_PROFILER
611 TCGProfile prof;
612#endif
613
614#ifdef CONFIG_DEBUG_TCG
615 int temps_in_use;
616 int goto_tb_issue_mask;
617 const TCGOpcode *vecop_list;
618#endif
619
620
621
622
623
624 void *code_gen_prologue;
625 void *code_gen_epilogue;
626 void *code_gen_buffer;
627 size_t code_gen_buffer_size;
628 void *code_gen_ptr;
629 void *data_gen_ptr;
630
631
632 void *code_gen_highwater;
633
634 size_t tb_phys_invalidate_count;
635
636
637 CPUState *cpu;
638
639
640#ifdef TCG_TARGET_NEED_LDST_LABELS
641 QSIMPLEQ_HEAD(, TCGLabelQemuLdst) ldst_labels;
642#endif
643#ifdef TCG_TARGET_NEED_POOL_LABELS
644 struct TCGLabelPoolData *pool_labels;
645#endif
646
647 TCGLabel *exitreq_label;
648
649#ifdef CONFIG_PLUGIN
650
651
652
653
654
655
656
657 struct qemu_plugin_tb *plugin_tb;
658
659
660 struct qemu_plugin_insn *plugin_insn;
661
662
663 QSIMPLEQ_HEAD(, TCGOp) plugin_ops;
664#endif
665
666 TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
667 TCGTemp temps[TCG_MAX_TEMPS];
668
669 QTAILQ_HEAD(, TCGOp) ops, free_ops;
670 QSIMPLEQ_HEAD(, TCGLabel) labels;
671
672
673
674 TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
675
676 uint16_t gen_insn_end_off[TCG_MAX_INSNS];
677 target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
678};
679
680extern TCGContext tcg_init_ctx;
681extern __thread TCGContext *tcg_ctx;
682extern TCGv_env cpu_env;
683
684static inline size_t temp_idx(TCGTemp *ts)
685{
686 ptrdiff_t n = ts - tcg_ctx->temps;
687 tcg_debug_assert(n >= 0 && n < tcg_ctx->nb_temps);
688 return n;
689}
690
691static inline TCGArg temp_arg(TCGTemp *ts)
692{
693 return (uintptr_t)ts;
694}
695
696static inline TCGTemp *arg_temp(TCGArg a)
697{
698 return (TCGTemp *)(uintptr_t)a;
699}
700
701
702
703
704static inline TCGTemp *tcgv_i32_temp(TCGv_i32 v)
705{
706 uintptr_t o = (uintptr_t)v;
707 TCGTemp *t = (void *)tcg_ctx + o;
708 tcg_debug_assert(offsetof(TCGContext, temps[temp_idx(t)]) == o);
709 return t;
710}
711
712static inline TCGTemp *tcgv_i64_temp(TCGv_i64 v)
713{
714 return tcgv_i32_temp((TCGv_i32)v);
715}
716
717static inline TCGTemp *tcgv_ptr_temp(TCGv_ptr v)
718{
719 return tcgv_i32_temp((TCGv_i32)v);
720}
721
722static inline TCGTemp *tcgv_vec_temp(TCGv_vec v)
723{
724 return tcgv_i32_temp((TCGv_i32)v);
725}
726
727static inline TCGArg tcgv_i32_arg(TCGv_i32 v)
728{
729 return temp_arg(tcgv_i32_temp(v));
730}
731
732static inline TCGArg tcgv_i64_arg(TCGv_i64 v)
733{
734 return temp_arg(tcgv_i64_temp(v));
735}
736
737static inline TCGArg tcgv_ptr_arg(TCGv_ptr v)
738{
739 return temp_arg(tcgv_ptr_temp(v));
740}
741
742static inline TCGArg tcgv_vec_arg(TCGv_vec v)
743{
744 return temp_arg(tcgv_vec_temp(v));
745}
746
747static inline TCGv_i32 temp_tcgv_i32(TCGTemp *t)
748{
749 (void)temp_idx(t);
750 return (TCGv_i32)((void *)t - (void *)tcg_ctx);
751}
752
753static inline TCGv_i64 temp_tcgv_i64(TCGTemp *t)
754{
755 return (TCGv_i64)temp_tcgv_i32(t);
756}
757
758static inline TCGv_ptr temp_tcgv_ptr(TCGTemp *t)
759{
760 return (TCGv_ptr)temp_tcgv_i32(t);
761}
762
763static inline TCGv_vec temp_tcgv_vec(TCGTemp *t)
764{
765 return (TCGv_vec)temp_tcgv_i32(t);
766}
767
768#if TCG_TARGET_REG_BITS == 32
769static inline TCGv_i32 TCGV_LOW(TCGv_i64 t)
770{
771 return temp_tcgv_i32(tcgv_i64_temp(t));
772}
773
774static inline TCGv_i32 TCGV_HIGH(TCGv_i64 t)
775{
776 return temp_tcgv_i32(tcgv_i64_temp(t) + 1);
777}
778#endif
779
780static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v)
781{
782 op->args[arg] = v;
783}
784
785static inline void tcg_set_insn_start_param(TCGOp *op, int arg, target_ulong v)
786{
787#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
788 tcg_set_insn_param(op, arg, v);
789#else
790 tcg_set_insn_param(op, arg * 2, v);
791 tcg_set_insn_param(op, arg * 2 + 1, v >> 32);
792#endif
793}
794
795
796static inline TCGOp *tcg_last_op(void)
797{
798 return QTAILQ_LAST(&tcg_ctx->ops);
799}
800
801
802static inline bool tcg_op_buf_full(void)
803{
804
805
806
807
808
809
810
811 return tcg_ctx->nb_ops >= 4000;
812}
813
814
815
816
817void *tcg_malloc_internal(TCGContext *s, int size);
818void tcg_pool_reset(TCGContext *s);
819TranslationBlock *tcg_tb_alloc(TCGContext *s);
820
821void tcg_region_init(void);
822void tb_destroy(TranslationBlock *tb);
823void tcg_region_reset_all(void);
824
825size_t tcg_code_size(void);
826size_t tcg_code_capacity(void);
827
828void tcg_tb_insert(TranslationBlock *tb);
829void tcg_tb_remove(TranslationBlock *tb);
830size_t tcg_tb_phys_invalidate_count(void);
831TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr);
832void tcg_tb_foreach(GTraverseFunc func, gpointer user_data);
833size_t tcg_nb_tbs(void);
834
835
836static inline void *tcg_malloc(int size)
837{
838 TCGContext *s = tcg_ctx;
839 uint8_t *ptr, *ptr_end;
840
841
842 size = QEMU_ALIGN_UP(size, 8);
843
844 ptr = s->pool_cur;
845 ptr_end = ptr + size;
846 if (unlikely(ptr_end > s->pool_end)) {
847 return tcg_malloc_internal(tcg_ctx, size);
848 } else {
849 s->pool_cur = ptr_end;
850 return ptr;
851 }
852}
853
854void tcg_context_init(TCGContext *s);
855void tcg_register_thread(void);
856void tcg_prologue_init(TCGContext *s);
857void tcg_func_start(TCGContext *s);
858
859int tcg_gen_code(TCGContext *s, TranslationBlock *tb);
860
861void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
862
863TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
864 intptr_t, const char *);
865TCGTemp *tcg_temp_new_internal(TCGType, bool);
866void tcg_temp_free_internal(TCGTemp *);
867TCGv_vec tcg_temp_new_vec(TCGType type);
868TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match);
869
870static inline void tcg_temp_free_i32(TCGv_i32 arg)
871{
872 tcg_temp_free_internal(tcgv_i32_temp(arg));
873}
874
875static inline void tcg_temp_free_i64(TCGv_i64 arg)
876{
877 tcg_temp_free_internal(tcgv_i64_temp(arg));
878}
879
880static inline void tcg_temp_free_ptr(TCGv_ptr arg)
881{
882 tcg_temp_free_internal(tcgv_ptr_temp(arg));
883}
884
885static inline void tcg_temp_free_vec(TCGv_vec arg)
886{
887 tcg_temp_free_internal(tcgv_vec_temp(arg));
888}
889
890static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
891 const char *name)
892{
893 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
894 return temp_tcgv_i32(t);
895}
896
897static inline TCGv_i32 tcg_temp_new_i32(void)
898{
899 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, false);
900 return temp_tcgv_i32(t);
901}
902
903static inline TCGv_i32 tcg_temp_local_new_i32(void)
904{
905 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, true);
906 return temp_tcgv_i32(t);
907}
908
909static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
910 const char *name)
911{
912 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
913 return temp_tcgv_i64(t);
914}
915
916static inline TCGv_i64 tcg_temp_new_i64(void)
917{
918 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, false);
919 return temp_tcgv_i64(t);
920}
921
922static inline TCGv_i64 tcg_temp_local_new_i64(void)
923{
924 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, true);
925 return temp_tcgv_i64(t);
926}
927
928static inline TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t offset,
929 const char *name)
930{
931 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_PTR, reg, offset, name);
932 return temp_tcgv_ptr(t);
933}
934
935static inline TCGv_ptr tcg_temp_new_ptr(void)
936{
937 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, false);
938 return temp_tcgv_ptr(t);
939}
940
941static inline TCGv_ptr tcg_temp_local_new_ptr(void)
942{
943 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, true);
944 return temp_tcgv_ptr(t);
945}
946
947#if defined(CONFIG_DEBUG_TCG)
948
949
950
951
952
953void tcg_clear_temp_count(void);
954int tcg_check_temp_count(void);
955#else
956#define tcg_clear_temp_count() do { } while (0)
957#define tcg_check_temp_count() 0
958#endif
959
960int64_t tcg_cpu_exec_time(void);
961void tcg_dump_info(void);
962void tcg_dump_op_count(void);
963
964#define TCG_CT_ALIAS 0x80
965#define TCG_CT_IALIAS 0x40
966#define TCG_CT_NEWREG 0x20
967#define TCG_CT_REG 0x01
968#define TCG_CT_CONST 0x02
969
970typedef struct TCGArgConstraint {
971 uint16_t ct;
972 uint8_t alias_index;
973 union {
974 TCGRegSet regs;
975 } u;
976} TCGArgConstraint;
977
978#define TCG_MAX_OP_ARGS 16
979
980
981enum {
982
983 TCG_OPF_BB_EXIT = 0x01,
984
985 TCG_OPF_BB_END = 0x02,
986
987 TCG_OPF_CALL_CLOBBER = 0x04,
988
989
990 TCG_OPF_SIDE_EFFECTS = 0x08,
991
992 TCG_OPF_64BIT = 0x10,
993
994
995 TCG_OPF_NOT_PRESENT = 0x20,
996
997 TCG_OPF_VECTOR = 0x40,
998};
999
1000typedef struct TCGOpDef {
1001 const char *name;
1002 uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
1003 uint8_t flags;
1004 TCGArgConstraint *args_ct;
1005 int *sorted_args;
1006#if defined(CONFIG_DEBUG_TCG)
1007 int used;
1008#endif
1009} TCGOpDef;
1010
1011extern TCGOpDef tcg_op_defs[];
1012extern const size_t tcg_op_defs_max;
1013
1014typedef struct TCGTargetOpDef {
1015 TCGOpcode op;
1016 const char *args_ct_str[TCG_MAX_OP_ARGS];
1017} TCGTargetOpDef;
1018
1019#define tcg_abort() \
1020do {\
1021 fprintf(stderr, "%s:%d: tcg fatal error\n", __FILE__, __LINE__);\
1022 abort();\
1023} while (0)
1024
1025bool tcg_op_supported(TCGOpcode op);
1026
1027void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args);
1028
1029TCGOp *tcg_emit_op(TCGOpcode opc);
1030void tcg_op_remove(TCGContext *s, TCGOp *op);
1031TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc);
1032TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc);
1033
1034void tcg_optimize(TCGContext *s);
1035
1036TCGv_i32 tcg_const_i32(int32_t val);
1037TCGv_i64 tcg_const_i64(int64_t val);
1038TCGv_i32 tcg_const_local_i32(int32_t val);
1039TCGv_i64 tcg_const_local_i64(int64_t val);
1040TCGv_vec tcg_const_zeros_vec(TCGType);
1041TCGv_vec tcg_const_ones_vec(TCGType);
1042TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec);
1043TCGv_vec tcg_const_ones_vec_matching(TCGv_vec);
1044
1045#if UINTPTR_MAX == UINT32_MAX
1046# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i32((intptr_t)(x)))
1047# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i32((intptr_t)(x)))
1048#else
1049# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i64((intptr_t)(x)))
1050# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i64((intptr_t)(x)))
1051#endif
1052
1053TCGLabel *gen_new_label(void);
1054
1055
1056
1057
1058
1059
1060
1061
1062static inline TCGArg label_arg(TCGLabel *l)
1063{
1064 return (uintptr_t)l;
1065}
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075static inline TCGLabel *arg_label(TCGArg i)
1076{
1077 return (TCGLabel *)(uintptr_t)i;
1078}
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
1094{
1095 return a - b;
1096}
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, void *target)
1108{
1109 return tcg_ptr_byte_diff(target, s->code_ptr);
1110}
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120static inline size_t tcg_current_code_size(TCGContext *s)
1121{
1122 return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
1123}
1124
1125
1126typedef uint32_t TCGMemOpIdx;
1127
1128
1129
1130
1131
1132
1133
1134
1135static inline TCGMemOpIdx make_memop_idx(MemOp op, unsigned idx)
1136{
1137 tcg_debug_assert(idx <= 15);
1138 return (op << 4) | idx;
1139}
1140
1141
1142
1143
1144
1145
1146
1147static inline MemOp get_memop(TCGMemOpIdx oi)
1148{
1149 return oi >> 4;
1150}
1151
1152
1153
1154
1155
1156
1157
1158static inline unsigned get_mmuidx(TCGMemOpIdx oi)
1159{
1160 return oi & 15;
1161}
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208#define TB_EXIT_MASK 3
1209#define TB_EXIT_IDX0 0
1210#define TB_EXIT_IDX1 1
1211#define TB_EXIT_IDXMAX 1
1212#define TB_EXIT_REQUESTED 3
1213
1214#ifdef HAVE_TCG_QEMU_TB_EXEC
1215uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
1216#else
1217# define tcg_qemu_tb_exec(env, tb_ptr) \
1218 ((uintptr_t (*)(void *, void *))tcg_ctx->code_gen_prologue)(env, tb_ptr)
1219#endif
1220
1221void tcg_register_jit(void *buf, size_t buf_size);
1222
1223#if TCG_TARGET_MAYBE_vec
1224
1225
1226
1227int tcg_can_emit_vec_op(TCGOpcode, TCGType, unsigned);
1228#else
1229static inline int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
1230{
1231 return 0;
1232}
1233#endif
1234
1235
1236void tcg_expand_vec_op(TCGOpcode, TCGType, unsigned, TCGArg, ...);
1237
1238
1239uint64_t dup_const(unsigned vece, uint64_t c);
1240
1241#define dup_const(VECE, C) \
1242 (__builtin_constant_p(VECE) \
1243 ? ( (VECE) == MO_8 ? 0x0101010101010101ull * (uint8_t)(C) \
1244 : (VECE) == MO_16 ? 0x0001000100010001ull * (uint16_t)(C) \
1245 : (VECE) == MO_32 ? 0x0000000100000001ull * (uint32_t)(C) \
1246 : dup_const(VECE, C)) \
1247 : dup_const(VECE, C))
1248
1249
1250
1251
1252
1253#ifdef CONFIG_SOFTMMU
1254
1255tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
1256 TCGMemOpIdx oi, uintptr_t retaddr);
1257tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
1258 TCGMemOpIdx oi, uintptr_t retaddr);
1259tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
1260 TCGMemOpIdx oi, uintptr_t retaddr);
1261uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
1262 TCGMemOpIdx oi, uintptr_t retaddr);
1263tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
1264 TCGMemOpIdx oi, uintptr_t retaddr);
1265tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
1266 TCGMemOpIdx oi, uintptr_t retaddr);
1267uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
1268 TCGMemOpIdx oi, uintptr_t retaddr);
1269
1270
1271tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
1272 TCGMemOpIdx oi, uintptr_t retaddr);
1273tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
1274 TCGMemOpIdx oi, uintptr_t retaddr);
1275tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
1276 TCGMemOpIdx oi, uintptr_t retaddr);
1277tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
1278 TCGMemOpIdx oi, uintptr_t retaddr);
1279tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
1280 TCGMemOpIdx oi, uintptr_t retaddr);
1281
1282void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
1283 TCGMemOpIdx oi, uintptr_t retaddr);
1284void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1285 TCGMemOpIdx oi, uintptr_t retaddr);
1286void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1287 TCGMemOpIdx oi, uintptr_t retaddr);
1288void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1289 TCGMemOpIdx oi, uintptr_t retaddr);
1290void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1291 TCGMemOpIdx oi, uintptr_t retaddr);
1292void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1293 TCGMemOpIdx oi, uintptr_t retaddr);
1294void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1295 TCGMemOpIdx oi, uintptr_t retaddr);
1296
1297
1298#ifdef TARGET_WORDS_BIGENDIAN
1299# define helper_ret_ldsw_mmu helper_be_ldsw_mmu
1300# define helper_ret_lduw_mmu helper_be_lduw_mmu
1301# define helper_ret_ldsl_mmu helper_be_ldsl_mmu
1302# define helper_ret_ldul_mmu helper_be_ldul_mmu
1303# define helper_ret_ldl_mmu helper_be_ldul_mmu
1304# define helper_ret_ldq_mmu helper_be_ldq_mmu
1305# define helper_ret_stw_mmu helper_be_stw_mmu
1306# define helper_ret_stl_mmu helper_be_stl_mmu
1307# define helper_ret_stq_mmu helper_be_stq_mmu
1308#else
1309# define helper_ret_ldsw_mmu helper_le_ldsw_mmu
1310# define helper_ret_lduw_mmu helper_le_lduw_mmu
1311# define helper_ret_ldsl_mmu helper_le_ldsl_mmu
1312# define helper_ret_ldul_mmu helper_le_ldul_mmu
1313# define helper_ret_ldl_mmu helper_le_ldul_mmu
1314# define helper_ret_ldq_mmu helper_le_ldq_mmu
1315# define helper_ret_stw_mmu helper_le_stw_mmu
1316# define helper_ret_stl_mmu helper_le_stl_mmu
1317# define helper_ret_stq_mmu helper_le_stq_mmu
1318#endif
1319
1320uint32_t helper_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
1321 uint32_t cmpv, uint32_t newv,
1322 TCGMemOpIdx oi, uintptr_t retaddr);
1323uint32_t helper_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
1324 uint32_t cmpv, uint32_t newv,
1325 TCGMemOpIdx oi, uintptr_t retaddr);
1326uint32_t helper_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
1327 uint32_t cmpv, uint32_t newv,
1328 TCGMemOpIdx oi, uintptr_t retaddr);
1329uint64_t helper_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
1330 uint64_t cmpv, uint64_t newv,
1331 TCGMemOpIdx oi, uintptr_t retaddr);
1332uint32_t helper_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
1333 uint32_t cmpv, uint32_t newv,
1334 TCGMemOpIdx oi, uintptr_t retaddr);
1335uint32_t helper_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
1336 uint32_t cmpv, uint32_t newv,
1337 TCGMemOpIdx oi, uintptr_t retaddr);
1338uint64_t helper_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
1339 uint64_t cmpv, uint64_t newv,
1340 TCGMemOpIdx oi, uintptr_t retaddr);
1341
1342#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
1343TYPE helper_atomic_ ## NAME ## SUFFIX ## _mmu \
1344 (CPUArchState *env, target_ulong addr, TYPE val, \
1345 TCGMemOpIdx oi, uintptr_t retaddr);
1346
1347#ifdef CONFIG_ATOMIC64
1348#define GEN_ATOMIC_HELPER_ALL(NAME) \
1349 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1350 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1351 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1352 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1353 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
1354 GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
1355 GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
1356#else
1357#define GEN_ATOMIC_HELPER_ALL(NAME) \
1358 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1359 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1360 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1361 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1362 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
1363#endif
1364
1365GEN_ATOMIC_HELPER_ALL(fetch_add)
1366GEN_ATOMIC_HELPER_ALL(fetch_sub)
1367GEN_ATOMIC_HELPER_ALL(fetch_and)
1368GEN_ATOMIC_HELPER_ALL(fetch_or)
1369GEN_ATOMIC_HELPER_ALL(fetch_xor)
1370GEN_ATOMIC_HELPER_ALL(fetch_smin)
1371GEN_ATOMIC_HELPER_ALL(fetch_umin)
1372GEN_ATOMIC_HELPER_ALL(fetch_smax)
1373GEN_ATOMIC_HELPER_ALL(fetch_umax)
1374
1375GEN_ATOMIC_HELPER_ALL(add_fetch)
1376GEN_ATOMIC_HELPER_ALL(sub_fetch)
1377GEN_ATOMIC_HELPER_ALL(and_fetch)
1378GEN_ATOMIC_HELPER_ALL(or_fetch)
1379GEN_ATOMIC_HELPER_ALL(xor_fetch)
1380GEN_ATOMIC_HELPER_ALL(smin_fetch)
1381GEN_ATOMIC_HELPER_ALL(umin_fetch)
1382GEN_ATOMIC_HELPER_ALL(smax_fetch)
1383GEN_ATOMIC_HELPER_ALL(umax_fetch)
1384
1385GEN_ATOMIC_HELPER_ALL(xchg)
1386
1387#undef GEN_ATOMIC_HELPER_ALL
1388#undef GEN_ATOMIC_HELPER
1389#endif
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
1400 Int128 cmpv, Int128 newv,
1401 TCGMemOpIdx oi, uintptr_t retaddr);
1402Int128 helper_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
1403 Int128 cmpv, Int128 newv,
1404 TCGMemOpIdx oi, uintptr_t retaddr);
1405
1406Int128 helper_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
1407 TCGMemOpIdx oi, uintptr_t retaddr);
1408Int128 helper_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
1409 TCGMemOpIdx oi, uintptr_t retaddr);
1410void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1411 TCGMemOpIdx oi, uintptr_t retaddr);
1412void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1413 TCGMemOpIdx oi, uintptr_t retaddr);
1414
1415#ifdef CONFIG_DEBUG_TCG
1416void tcg_assert_listed_vecop(TCGOpcode);
1417#else
1418static inline void tcg_assert_listed_vecop(TCGOpcode op) { }
1419#endif
1420
1421static inline const TCGOpcode *tcg_swap_vecop_list(const TCGOpcode *n)
1422{
1423#ifdef CONFIG_DEBUG_TCG
1424 const TCGOpcode *o = tcg_ctx->vecop_list;
1425 tcg_ctx->vecop_list = n;
1426 return o;
1427#else
1428 return NULL;
1429#endif
1430}
1431
1432bool tcg_can_emit_vecop_list(const TCGOpcode *, TCGType, unsigned);
1433
1434#endif
1435